From 3759bafe2dd7edbb32ed6957d595eda4ce757b93 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 5 May 2022 14:52:58 -0400 Subject: [PATCH 01/27] add tinkerbell ci test harness to e2e test framework --- Makefile | 34 ++-- .../build/buildspecs/test-eks-a-cli.yml | 2 +- .../build/buildspecs/test-infra.yml | 16 ++ cmd/integration_test/cmd/run.go | 42 ++--- internal/pkg/ssm/activation.go | 30 +++ internal/pkg/ssm/instance.go | 31 +++ internal/pkg/vsphere/deploy.go | 56 ++++++ internal/test/e2e/run.go | 78 +++++--- internal/test/e2e/setup.go | 35 ++-- internal/test/e2e/testRunner.go | 177 ++++++++++++++++++ internal/test/e2e/tinkerbell.go | 30 +++ pkg/executables/govc.go | 12 +- test/framework/tinkerbell.go | 4 + 13 files changed, 461 insertions(+), 86 deletions(-) create mode 100644 cmd/integration_test/build/buildspecs/test-infra.yml create mode 100644 internal/pkg/ssm/activation.go create mode 100644 internal/pkg/ssm/instance.go create mode 100644 internal/pkg/vsphere/deploy.go create mode 100644 internal/test/e2e/testRunner.go create mode 100644 internal/test/e2e/tinkerbell.go diff --git a/Makefile b/Makefile index eba19141da58..4411f7c684fa 100644 --- a/Makefile +++ b/Makefile @@ -118,6 +118,7 @@ ORGANIZE_BINARIES_TARGETS = $(addsuffix /eks-a-tools,$(addprefix $(BINARY_DEPS_D EKS_A_PLATFORMS ?= linux-amd64 linux-arm64 darwin-arm64 darwin-amd64 EKS_A_CROSS_PLATFORMS := $(foreach platform,$(EKS_A_PLATFORMS),eks-a-cross-platform-$(platform)) +E2E_CROSS_PLATFORMS := $(foreach platform,$(EKS_A_PLATFORMS),e2e-cross-platform-$(platform)) EKS_A_RELEASE_CROSS_PLATFORMS := $(foreach platform,$(EKS_A_PLATFORMS),eks-a-release-cross-platform-$(platform)) DOCKER_E2E_TEST := TestDockerKubernetes121SimpleFlow @@ -465,19 +466,20 @@ verify-mocks: mocks ## Verify if mocks need to be updated exit 1;\ fi +.PHONY: e2e-cross-platform +e2e-cross-platform: $(E2E_CROSS_PLATFORMS) + +.PHONY: e2e-cross-platform-% +e2e-cross-platform-%: ## Generate binaries for Linux and MacOS +e2e-cross-platform-%: GO_OS = $(firstword $(subst -, ,$*)) +e2e-cross-platform-%: GO_ARCH = $(lastword $(subst -, ,$*)) +e2e-cross-platform-%: + $(MAKE) e2e-tests-binary E2E_TAGS=e2e GIT_VERSION=$(DEV_GIT_VERSION) GO_OS=$(GO_OS) GO_ARCH=$(GO_ARCH) E2E_OUTPUT_FILE=bin/$(GO_OS)/$(GO_ARCH)/e2e.test + .PHONY: e2e e2e: eks-a-e2e integration-test-binary ## Build integration tests $(MAKE) e2e-tests-binary E2E_TAGS=e2e -.PHONY: conformance -conformance: - $(MAKE) e2e-tests-binary E2E_TAGS=conformance_e2e - ./bin/e2e.test -test.v -test.run 'TestVSphereKubernetes121ThreeWorkersConformanc.*' - -.PHONY: conformance-tests -conformance-tests: eks-a-e2e integration-test-binary ## Build e2e conformance tests - $(MAKE) e2e-tests-binary E2E_TAGS=conformance_e2e - .PHONY: eks-a-e2e eks-a-e2e: if [ "$(CODEBUILD_CI)" = "true" ]; then \ @@ -496,12 +498,22 @@ eks-a-e2e: fi .PHONY: e2e-tests-binary +e2e-tests-binary: E2E_OUTPUT_FILE ?= bin/e2e.test e2e-tests-binary: - $(GO) test ./test/e2e -c -o bin/e2e.test -tags "$(E2E_TAGS)" -ldflags "-X github.com/aws/eks-anywhere/pkg/version.gitVersion=$(DEV_GIT_VERSION) -X github.com/aws/eks-anywhere/pkg/cluster.releasesManifestURL=$(RELEASE_MANIFEST_URL) -X github.com/aws/eks-anywhere/pkg/manifests/releases.manifestURL=$(RELEASE_MANIFEST_URL)" + GOOS=$(GO_OS) GOARCH=$(GO_ARCH) $(GO) test ./test/e2e -c -o bin/e2e.test -tags "$(E2E_TAGS)" -ldflags "-X github.com/aws/eks-anywhere/pkg/version.gitVersion=$(DEV_GIT_VERSION) -X github.com/aws/eks-anywhere/pkg/cluster.releasesManifestURL=$(RELEASE_MANIFEST_URL) -X github.com/aws/eks-anywhere/pkg/manifests/releases.manifestURL=$(RELEASE_MANIFEST_URL)" .PHONY: integration-test-binary integration-test-binary: - $(GO) build -o bin/test github.com/aws/eks-anywhere/cmd/integration_test + GOOS=$(GO_OS) GOARCH=$(GO_ARCH) $(GO) build -o bin/test github.com/aws/eks-anywhere/cmd/integration_test + +.PHONY: conformance +conformance: + $(MAKE) e2e-tests-binary E2E_TAGS=conformance_e2e + ./bin/e2e.test -test.v -test.run 'TestVSphereKubernetes121ThreeWorkersConformanc.*' + +.PHONY: conformance-tests +conformance-tests: eks-a-e2e integration-test-binary ## Build e2e conformance tests + $(MAKE) e2e-tests-binary E2E_TAGS=conformance_e2e .PHONY: check-eksa-components-override check-eksa-components-override: diff --git a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml index 9d1a2556fd5b..e8b2bcd617ed 100644 --- a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml @@ -92,7 +92,7 @@ phases: -i ${INTEGRATION_TEST_INSTANCE_PROFILE} -n ${INTEGRATION_TEST_SUBNET_ID} -m ${INTEGRATION_TEST_MAX_EC2_COUNT} - -c ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT} + -p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT} -r 'Test' -v 4 --skip ${SKIPPED_TESTS} diff --git a/cmd/integration_test/build/buildspecs/test-infra.yml b/cmd/integration_test/build/buildspecs/test-infra.yml new file mode 100644 index 000000000000..2b6b30bc5b3f --- /dev/null +++ b/cmd/integration_test/build/buildspecs/test-infra.yml @@ -0,0 +1,16 @@ +--- + +ec2: + amiId: ami-00c71c2441b729da0 + subnetId: subnet-0fd4b6ba7c9e17b5f + +vSphere: + url: https://10.61.250.74 + insecure: True + library: eks-a-templates + template: eks-a-admin-v0.0.0-8a498c1-20220511153117 + datacenter: Datacenter + datastore: datastore2 + resourcePool: TestResourcePool + network: VM Network + folder: gwesterf \ No newline at end of file diff --git a/cmd/integration_test/cmd/run.go b/cmd/integration_test/cmd/run.go index 56e11484a340..a26a367c116f 100644 --- a/cmd/integration_test/cmd/run.go +++ b/cmd/integration_test/cmd/run.go @@ -14,11 +14,9 @@ import ( ) const ( - amiIdFlagName = "ami-id" storageBucketFlagName = "storage-bucket" jobIdFlagName = "job-id" instanceProfileFlagName = "instance-profile-name" - subnetIdFlagName = "subnet-id" regexFlagName = "regex" maxInstancesFlagName = "max-instances" maxConcurrentTestsFlagName = "max-concurrent-tests" @@ -27,6 +25,7 @@ const ( cleanupVmsFlagName = "cleanup-vms" testReportFolderFlagName = "test-report-folder" branchNameFlagName = "branch-name" + instanceConfigFlagName = "instance-config" ) var runE2ECmd = &cobra.Command{ @@ -44,7 +43,7 @@ var runE2ECmd = &cobra.Command{ }, } -var requiredFlags = []string{amiIdFlagName, storageBucketFlagName, jobIdFlagName, instanceProfileFlagName} +var requiredFlags = []string{instanceConfigFlagName, storageBucketFlagName, jobIdFlagName, instanceProfileFlagName} func preRunSetup(cmd *cobra.Command, args []string) { cmd.Flags().VisitAll(func(flag *pflag.Flag) { @@ -57,18 +56,17 @@ func preRunSetup(cmd *cobra.Command, args []string) { func init() { integrationTestCmd.AddCommand(runE2ECmd) - runE2ECmd.Flags().StringP(amiIdFlagName, "a", "", "Ami id") + runE2ECmd.Flags().StringP(instanceConfigFlagName, "c", "", "File path to the instance-config.yml config") runE2ECmd.Flags().StringP(storageBucketFlagName, "s", "", "S3 bucket name to store eks-a binary") runE2ECmd.Flags().StringP(jobIdFlagName, "j", "", "Id of the job being run") - runE2ECmd.Flags().StringP(instanceProfileFlagName, "i", "", "IAM instance profile name to attach to ec2 instances") - runE2ECmd.Flags().StringP(subnetIdFlagName, "n", "", "EC2 subnet ID") + runE2ECmd.Flags().StringP(instanceProfileFlagName, "i", "", "IAM instance profile name to attach to ssm instances") runE2ECmd.Flags().StringP(regexFlagName, "r", "", "Run only those tests and examples matching the regular expression. Equivalent to go test -run") runE2ECmd.Flags().IntP(maxInstancesFlagName, "m", 1, "Run tests in parallel on same instance within the max EC2 instance count") - runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "c", 1, "Maximum number of parallel tests that can be run at a time") + runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "p", 1, "Maximum number of parallel tests that can be run at a time") runE2ECmd.Flags().StringSlice(skipFlagName, nil, "List of tests to skip") runE2ECmd.Flags().Bool(bundlesOverrideFlagName, false, "Flag to indicate if the tests should run with a bundles override") runE2ECmd.Flags().Bool(cleanupVmsFlagName, false, "Flag to indicate if VSphere VMs should be cleaned up automatically as tests complete") - runE2ECmd.Flags().String(testReportFolderFlagName, "", "Folder destination fo JUnit tests reports") + runE2ECmd.Flags().String(testReportFolderFlagName, "", "Folder destination for JUnit tests reports") runE2ECmd.Flags().String(branchNameFlagName, "main", "EKS-A origin branch from where the tests are being run") for _, flag := range requiredFlags { @@ -79,11 +77,10 @@ func init() { } func runE2E(ctx context.Context) error { - amiId := viper.GetString(amiIdFlagName) + instanceConfigFile := viper.GetString(instanceConfigFlagName) storageBucket := viper.GetString(storageBucketFlagName) jobId := viper.GetString(jobIdFlagName) instanceProfileName := viper.GetString(instanceProfileFlagName) - subnetId := viper.GetString(subnetIdFlagName) testRegex := viper.GetString(regexFlagName) maxInstances := viper.GetInt(maxInstancesFlagName) maxConcurrentTests := viper.GetInt(maxConcurrentTestsFlagName) @@ -94,19 +91,18 @@ func runE2E(ctx context.Context) error { branchName := viper.GetString(branchNameFlagName) runConf := e2e.ParallelRunConf{ - MaxInstances: maxInstances, - MaxConcurrentTests: maxConcurrentTests, - AmiId: amiId, - InstanceProfileName: instanceProfileName, - StorageBucket: storageBucket, - JobId: jobId, - SubnetId: subnetId, - Regex: testRegex, - TestsToSkip: testsToSkip, - BundlesOverride: bundlesOverride, - CleanupVms: cleanupVms, - TestReportFolder: testReportFolder, - BranchName: branchName, + MaxInstances: maxInstances, + MaxConcurrentTests: maxConcurrentTests, + InstanceProfileName: instanceProfileName, + StorageBucket: storageBucket, + JobId: jobId, + Regex: testRegex, + TestsToSkip: testsToSkip, + BundlesOverride: bundlesOverride, + CleanupVms: cleanupVms, + TestReportFolder: testReportFolder, + BranchName: branchName, + TestInstanceConfigFile: instanceConfigFile, } err := e2e.RunTestsInParallel(runConf) diff --git a/internal/pkg/ssm/activation.go b/internal/pkg/ssm/activation.go new file mode 100644 index 000000000000..e3c8954867e6 --- /dev/null +++ b/internal/pkg/ssm/activation.go @@ -0,0 +1,30 @@ +package ssm + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ssm" +) + +type ActivationInfo struct { + ActivationCode string + ActivationID string +} + +func CreateActivation(session *session.Session, instanceName, role string) (*ActivationInfo, error) { + s := ssm.New(session) + + request := ssm.CreateActivationInput{ + DefaultInstanceName: &instanceName, + Description: &instanceName, + IamRole: &role, + } + + result, err := s.CreateActivation(&request) + if err != nil { + return nil, fmt.Errorf("failed to activate ssm instance %s: %v", instanceName, err) + } + + return &ActivationInfo{ActivationCode: *result.ActivationCode, ActivationID: *result.ActivationId}, nil +} diff --git a/internal/pkg/ssm/instance.go b/internal/pkg/ssm/instance.go new file mode 100644 index 000000000000..5beae2358414 --- /dev/null +++ b/internal/pkg/ssm/instance.go @@ -0,0 +1,31 @@ +package ssm + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ssm" +) + +func GetInstanceByActivationId(session *session.Session, id string) (*ssm.InstanceInformation, error) { + s := ssm.New(session) + instanceActivationIdKey := "ActivationIds" + input := ssm.DescribeInstanceInformationInput{ + Filters: []*ssm.InstanceInformationStringFilter{ + {Key: &instanceActivationIdKey, Values: []*string{&id}}, + }, + } + + output, err := s.DescribeInstanceInformation(&input) + if err != nil { + return nil, fmt.Errorf("failed to describe ssm instance %s: %v", id, err) + } + + infoList := output.InstanceInformationList + + if len(infoList) == 0 { + return nil, fmt.Errorf("no ssm instance with name %s: %v", id, err) + } + + return infoList[0], nil +} diff --git a/internal/pkg/vsphere/deploy.go b/internal/pkg/vsphere/deploy.go new file mode 100644 index 000000000000..a7e652c4d516 --- /dev/null +++ b/internal/pkg/vsphere/deploy.go @@ -0,0 +1,56 @@ +package vsphere + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/aws/eks-anywhere/pkg/executables" + "github.com/aws/eks-anywhere/pkg/filewriter" +) + +type OVFDeployOptions struct { + Name string `json:"Name"` + PowerOn bool `json:"PowerOn"` + DiskProvisioning string `json:"DiskProvisioning"` + WaitForIP bool `json:"WaitForIP"` + NetworkMappings []NetworkMapping `json:"NetworkMapping"` + Annotation string `json:"Annotation"` + PropertyMapping []OVFProperty `json:"PropertyMapping"` + InjectOvfEnv bool `json:"InjectOvfEnv"` +} + +type OVFProperty struct { + Key string `json:"Key"` + Value string `json:"Value"` +} + +type NetworkMapping struct { + Name string `json:"Name"` + Network string `json:"Network"` +} + +func DeployTemplate(library, templateName, vmName, deployFolder, datacenter, datastore, resourcePool string, opts OVFDeployOptions) error { + context := context.Background() + executableBuilder, close, err := executables.NewExecutableBuilder(context, executables.DefaultEksaImage()) + if err != nil { + return fmt.Errorf("unable to initialize executables: %v", err) + } + + defer close.CheckErr(context) + tmpWriter, _ := filewriter.NewWriter(vmName) + govc := executableBuilder.BuildGovcExecutable(tmpWriter) + defer govc.Close(context) + + deployOptions, err := json.Marshal(opts) + if err != nil { + return fmt.Errorf("failed to marshall vm deployment options: %v", err) + } + + // deploy template + if err := govc.DeployTemplate(context, library, templateName, vmName, deployFolder, datacenter, datastore, resourcePool, deployOptions); err != nil { + return fmt.Errorf("failed to deploy vm from library template: %v", err) + } + + return nil +} diff --git a/internal/test/e2e/run.go b/internal/test/e2e/run.go index 3dcf02426dee..abfdfd588050 100644 --- a/internal/test/e2e/run.go +++ b/internal/test/e2e/run.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/eks-anywhere/internal/pkg/ec2" "github.com/aws/eks-anywhere/internal/pkg/ssm" "github.com/aws/eks-anywhere/pkg/logger" @@ -22,19 +23,18 @@ const ( ) type ParallelRunConf struct { - MaxInstances int - MaxConcurrentTests int - AmiId string - InstanceProfileName string - StorageBucket string - JobId string - SubnetId string - Regex string - TestsToSkip []string - BundlesOverride bool - CleanupVms bool - TestReportFolder string - BranchName string + TestInstanceConfigFile string + MaxInstances int + MaxConcurrentTests int + InstanceProfileName string + StorageBucket string + JobId string + Regex string + TestsToSkip []string + BundlesOverride bool + CleanupVms bool + TestReportFolder string + BranchName string } type ( @@ -62,7 +62,11 @@ func RunTestsInParallel(conf ParallelRunConf) error { var wg sync.WaitGroup - instancesConf := splitTests(testsList, conf) + instancesConf, err := splitTests(testsList, conf) + if err != nil { + return fmt.Errorf("failed to split tests: %v", err) + } + results := make([]instanceTestsResults, 0, len(instancesConf)) logTestGroups(instancesConf) maxConcurrentTests := conf.MaxConcurrentTests @@ -111,15 +115,24 @@ func RunTestsInParallel(conf ParallelRunConf) error { } type instanceRunConf struct { - amiId, instanceProfileName, storageBucket, jobId, parentJobId, subnetId, regex, instanceId string - testReportFolder, branchName string - ipPool networkutils.IPPool - bundlesOverride bool - cleanupVms bool + session *session.Session + instanceProfileName, storageBucket, jobId, parentJobId, regex, instanceId string + testReportFolder, branchName string + ipPool networkutils.IPPool + bundlesOverride bool + testRunnerType TestRunnerType + testRunnerConfig TestInfraConfig + cleanupVms bool } func RunTests(conf instanceRunConf) (testInstanceID string, testCommandResult *testCommandResult, err error) { - session, err := newSessionFromConf(conf) + testRunner := newTestRunner(conf.testRunnerType, conf.testRunnerConfig) + instanceId, err := testRunner.createInstance(conf) + if err != nil { + return "", nil, err + } + + session, err := newE2ESession(instanceId, conf) if err != nil { return "", nil, err } @@ -200,13 +213,14 @@ func (e *E2ESession) commandWithEnvVars(command string) string { return strings.Join(fullCommand, "; ") } -func splitTests(testsList []string, conf ParallelRunConf) []instanceRunConf { +func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, error) { testPerInstance := len(testsList) / conf.MaxInstances if testPerInstance == 0 { testPerInstance = 1 } vsphereTestsRe := regexp.MustCompile(vsphereRegex) + tinkerbellTestsRe := regexp.MustCompile(tinkerbellTestsRe) privateNetworkTestsRe := regexp.MustCompile(`^.*(Proxy|RegistryMirror).*$`) multiClusterTestsRe := regexp.MustCompile(`^.*Multicluster.*$`) @@ -232,27 +246,43 @@ func splitTests(testsList []string, conf ParallelRunConf) []instanceRunConf { } } + awsSession, err := session.NewSession() + if err != nil { + return nil, fmt.Errorf("creating aws session for test: %s, %v", testName, err) + } + + testRunnerConfig, err := NewTestRunnerConfigFromFile(conf.TestInstanceConfigFile) + if err != nil { + return nil, fmt.Errorf("creating test runner config for test: %s, %v", testName, err) + } + + testRunnerType := Ec2TestRunnerType + if tinkerbellTestsRe.MatchString(testName) { + testRunnerType = VSphereTestRunnerType + } + if len(testsInCurrentInstance) == testPerInstance || (len(testsList)-1) == i { runConfs = append(runConfs, instanceRunConf{ - amiId: conf.AmiId, + session: awsSession, instanceProfileName: conf.InstanceProfileName, storageBucket: conf.StorageBucket, jobId: fmt.Sprintf("%s-%d", conf.JobId, len(runConfs)), parentJobId: conf.JobId, - subnetId: conf.SubnetId, regex: strings.Join(testsInCurrentInstance, "|"), bundlesOverride: conf.BundlesOverride, cleanupVms: conf.CleanupVms, testReportFolder: conf.TestReportFolder, branchName: conf.BranchName, ipPool: ips, + testRunnerType: testRunnerType, + testRunnerConfig: *testRunnerConfig, }) testsInCurrentInstance = make([]string, 0, testPerInstance) } } - return runConfs + return runConfs, nil } func logTestGroups(instancesConf []instanceRunConf) { diff --git a/internal/test/e2e/setup.go b/internal/test/e2e/setup.go index ae37a28f11b2..82e15d0c3f90 100644 --- a/internal/test/e2e/setup.go +++ b/internal/test/e2e/setup.go @@ -9,7 +9,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/eks-anywhere/internal/pkg/ec2" "github.com/aws/eks-anywhere/internal/pkg/s3" "github.com/aws/eks-anywhere/internal/pkg/ssm" "github.com/aws/eks-anywhere/pkg/logger" @@ -27,15 +26,15 @@ const ( testNameFile = "e2e-test-name" maxUserWatches = 524288 maxUserInstances = 512 + key = "Integration-Test" + tag = "EKSA-E2E" ) type E2ESession struct { session *session.Session - amiId string instanceProfileName string storageBucket string jobId string - subnetId string instanceId string ipPool networkutils.IPPool testEnvVars map[string]string @@ -45,19 +44,13 @@ type E2ESession struct { branchName string } -func newSessionFromConf(conf instanceRunConf) (*E2ESession, error) { - session, err := session.NewSession() - if err != nil { - return nil, fmt.Errorf("creating session: %v", err) - } - +func newE2ESession(instanceId string, conf instanceRunConf) (*E2ESession, error) { e := &E2ESession{ - session: session, - amiId: conf.amiId, + session: conf.session, + instanceId: instanceId, instanceProfileName: conf.instanceProfileName, storageBucket: conf.storageBucket, jobId: conf.jobId, - subnetId: conf.subnetId, ipPool: conf.ipPool, testEnvVars: make(map[string]string), bundlesOverride: conf.bundlesOverride, @@ -75,19 +68,8 @@ func (e *E2ESession) setup(regex string) error { return err } - key := "Integration-Test" - tag := "EKSA-E2E" - name := fmt.Sprintf("eksa-e2e-%s", e.jobId) - logger.V(1).Info("Creating ec2 instance", "name", name) - instanceId, err := ec2.CreateInstance(e.session, e.amiId, key, tag, e.instanceProfileName, e.subnetId, name) - if err != nil { - return fmt.Errorf("creating instance for e2e tests: %v", err) - } - logger.V(1).Info("Instance created", "instance-id", instanceId) - e.instanceId = instanceId - logger.V(1).Info("Waiting until SSM is ready") - err = ssm.WaitForSSMReady(e.session, instanceId) + err = ssm.WaitForSSMReady(e.session, e.instanceId) if err != nil { return fmt.Errorf("waiting for ssm in new instance: %v", err) } @@ -116,6 +98,11 @@ func (e *E2ESession) setup(regex string) error { return err } + err = e.setupTinkerbellEnv(regex) + if err != nil { + return err + } + err = e.setupCloudStackEnv(regex) if err != nil { return err diff --git a/internal/test/e2e/testRunner.go b/internal/test/e2e/testRunner.go new file mode 100644 index 000000000000..c8df89a2196c --- /dev/null +++ b/internal/test/e2e/testRunner.go @@ -0,0 +1,177 @@ +package e2e + +import ( + "fmt" + "os" + "strconv" + "time" + + aws_ssm "github.com/aws/aws-sdk-go/service/ssm" + "github.com/aws/eks-anywhere/internal/pkg/ec2" + "github.com/aws/eks-anywhere/internal/pkg/ssm" + "github.com/aws/eks-anywhere/internal/pkg/vsphere" + "github.com/aws/eks-anywhere/pkg/logger" + "github.com/aws/eks-anywhere/pkg/retrier" + "gopkg.in/yaml.v2" +) + +const ( + testRunnerVCUserEnvVar string = "TEST_RUNNER_GOVC_USERNAME" + testRunnerVCPasswordEnvVar string = "TEST_RUNNER_GOVC_PASSWORD" + govcUsernameKey string = "GOVC_USERNAME" + govcPasswordKey string = "GOVC_PASSWORD" + govcURLKey string = "GOVC_URL" + govcInsecure string = "GOVC_INSECURE" + ssmActivationCodeKey string = "ssm_activation_code" + ssmActivationIdKey string = "ssm_activation_id" + ssmActivationRegionKey string = "ssm_activation_region" +) + +type TestRunner interface { + createInstance(instanceConf instanceRunConf) (string, error) +} + +type TestRunnerType string + +const ( + Ec2TestRunnerType TestRunnerType = "ec2" + VSphereTestRunnerType TestRunnerType = "vSphere" +) + +func newTestRunner(runnerType TestRunnerType, config TestInfraConfig) TestRunner { + if runnerType == VSphereTestRunnerType { + return config.VSphereTestRunner + } else { + return config.Ec2TestRunner + } +} + +type TestInfraConfig struct { + Ec2TestRunner `yaml:"ec2,omitempty"` + VSphereTestRunner `yaml:"vSphere,omitempty"` +} + +func NewTestRunnerConfigFromFile(configFile string) (*TestInfraConfig, error) { + file, err := os.ReadFile(configFile) + if err != nil { + return nil, fmt.Errorf("failed to create test runner config from file: %v", err) + } + + config := TestInfraConfig{} + + err = yaml.Unmarshal(file, &config) + if err != nil { + return nil, fmt.Errorf("failed to create test runner config from file: %v", err) + } + + return &config, nil +} + +type Ec2TestRunner struct { + AmiID string `yaml:"amiId"` + SubnetID string `yaml:"subnetId"` +} + +type VSphereTestRunner struct { + Url string `yaml:"url"` + Insecure bool `yaml:"insecure"` + Library string `yaml:"library"` + Template string `yaml:"template"` + Datacenter string `yaml:"datacenter"` + Datastore string `yaml:"datastore"` + ResourcePool string `yaml:"resourcePool"` + Network string `yaml:"network"` + Folder string `yaml:"folder"` +} + +func (v *VSphereTestRunner) setEnvironment() error { + if vSphereUsername, ok := os.LookupEnv(testRunnerVCUserEnvVar); ok && len(vSphereUsername) > 0 { + if err := os.Setenv(govcUsernameKey, vSphereUsername); err != nil { + return fmt.Errorf("unable to set %s: %v", govcUsernameKey, err) + } + } else { + return fmt.Errorf("missing environment variable: %s", testRunnerVCUserEnvVar) + } + + if vSpherePassword, ok := os.LookupEnv(testRunnerVCPasswordEnvVar); ok && len(vSpherePassword) > 0 { + if err := os.Setenv(govcPasswordKey, vSpherePassword); err != nil { + return fmt.Errorf("unable to set %s: %v", govcPasswordKey, err) + } + } else { + return fmt.Errorf("missing environment variable: %s", testRunnerVCPasswordEnvVar) + } + + if err := os.Setenv(govcURLKey, v.Url); err != nil { + return fmt.Errorf("unable to set %s: %v", govcURLKey, err) + } + + if err := os.Setenv(govcInsecure, strconv.FormatBool(v.Insecure)); err != nil { + return fmt.Errorf("unable to set %s: %v", govcURLKey, err) + } + + return nil +} + +func (v VSphereTestRunner) createInstance(c instanceRunConf) (string, error) { + name := getTestRunnerName(c.jobId) + logger.V(1).Info("Creating vSphere Test Runner instance", "name", name) + + err := v.setEnvironment() + if err != nil { + return "", fmt.Errorf("unable to create vSphere test runner instance: %v", err) + } + + ssmActivationInfo, err := ssm.CreateActivation(c.session, name, c.instanceProfileName) + if err != nil { + return "", fmt.Errorf("unable to create ssm activation: %v", err) + } + + // import ova template from url if not exist + opts := vsphere.OVFDeployOptions{ + Name: name, + PowerOn: true, + DiskProvisioning: "thin", + WaitForIP: true, + InjectOvfEnv: true, + NetworkMappings: []vsphere.NetworkMapping{{Name: v.Network, Network: v.Network}}, + PropertyMapping: []vsphere.OVFProperty{ + {Key: ssmActivationCodeKey, Value: ssmActivationInfo.ActivationCode}, + {Key: ssmActivationIdKey, Value: ssmActivationInfo.ActivationID}, + {Key: ssmActivationRegionKey, Value: *c.session.Config.Region}, + }, + } + + // deploy template + if err := vsphere.DeployTemplate(v.Library, v.Template, name, v.Folder, v.Datacenter, v.Datastore, v.ResourcePool, opts); err != nil { + return "", err + } + + var ssmInstance *aws_ssm.InstanceInformation + err = retrier.Retry(10, 5*time.Second, func() error { + ssmInstance, err = ssm.GetInstanceByActivationId(c.session, ssmActivationInfo.ActivationID) + if err != nil { + return fmt.Errorf("failed to get ssm instance info post ovf deployment: %v", err) + } + return nil + }) + if err != nil { + return "", fmt.Errorf("waiting for ssm instance to activate %s : %v", name, err) + } + + return *ssmInstance.InstanceId, nil +} + +func (e Ec2TestRunner) createInstance(c instanceRunConf) (string, error) { + name := getTestRunnerName(c.jobId) + logger.V(1).Info("Creating ec2 Test Runner instance", "name", name) + instanceId, err := ec2.CreateInstance(c.session, e.AmiID, key, tag, c.instanceProfileName, e.SubnetID, name) + if err != nil { + return "", fmt.Errorf("creating instance for e2e tests: %v", err) + } + logger.V(1).Info("Instance created", "instance-id", instanceId) + return instanceId, nil +} + +func getTestRunnerName(jobId string) string { + return fmt.Sprintf("eksa-e2e-%s", jobId) +} diff --git a/internal/test/e2e/tinkerbell.go b/internal/test/e2e/tinkerbell.go new file mode 100644 index 000000000000..ea869497d6cd --- /dev/null +++ b/internal/test/e2e/tinkerbell.go @@ -0,0 +1,30 @@ +package e2e + +import ( + "os" + "regexp" + + "github.com/aws/eks-anywhere/pkg/logger" + e2etests "github.com/aws/eks-anywhere/test/framework" +) + +const ( + tinkerbellTestsRe = `^.*Tinkerbell.*$` +) + +func (e *E2ESession) setupTinkerbellEnv(testRegex string) error { + re := regexp.MustCompile(tinkerbellTestsRe) + if !re.MatchString(testRegex) { + logger.V(2).Info("Not running Tinkerbell tests, skipping Env variable setup") + return nil + } + + requiredEnvVars := e2etests.RequiredTinkerbellEnvVars() + for _, eVar := range requiredEnvVars { + if val, ok := os.LookupEnv(eVar); ok { + e.testEnvVars[eVar] = val + } + } + + return nil +} diff --git a/pkg/executables/govc.go b/pkg/executables/govc.go index 074ef70b439b..72712af6224b 100644 --- a/pkg/executables/govc.go +++ b/pkg/executables/govc.go @@ -278,7 +278,8 @@ func (g *Govc) CreateLibrary(ctx context.Context, datastore, library string) err func (g *Govc) DeployTemplateFromLibrary(ctx context.Context, templateDir, templateName, library, datacenter, datastore, network, resourcePool string, resizeBRDisk bool) error { logger.V(4).Info("Deploying template", "dir", templateDir, "templateName", templateName) - if err := g.deployTemplate(ctx, library, templateName, templateDir, datacenter, datastore, network, resourcePool); err != nil { + + if err := g.DeployTemplate(ctx, library, templateName, templateName, templateDir, datacenter, datastore, network, resourcePool, nil); err != nil { return err } @@ -350,7 +351,7 @@ func (g *Govc) ImportTemplate(ctx context.Context, library, ovaURL, name string) return nil } -func (g *Govc) deployTemplate(ctx context.Context, library, templateName, deployFolder, datacenter, datastore, network, resourcePool string) error { +func (g *Govc) DeployTemplate(ctx context.Context, library, templateName, vmName, deployFolder, datacenter, datastore, network, resourcePool string, deployOptionsOverride []byte) error { envMap, err := g.validateAndSetupCreds() if err != nil { return fmt.Errorf("failed govc validations: %v", err) @@ -366,7 +367,12 @@ func (g *Govc) deployTemplate(ctx context.Context, library, templateName, deploy return err } + if len(deployOptionsOverride) > 0 { + deployOpts = deployOptionsOverride + } + deployOptsPath, err := g.writer.Write(DeployOptsFile, deployOpts, filewriter.PersistentFile) + if err != nil { return fmt.Errorf("failed writing deploy options file to disk: %v", err) } @@ -407,7 +413,7 @@ func (g *Govc) deployTemplate(ctx context.Context, library, templateName, deploy "-pool", resourcePool, "-folder", deployFolder, "-options", deployOptsPath, - templateInLibraryPath, templateName, + templateInLibraryPath, vmName, } if _, err := g.exec(ctx, params...); err != nil { return fmt.Errorf("deploying template: %v", err) diff --git a/test/framework/tinkerbell.go b/test/framework/tinkerbell.go index 65eb4b7a3720..2a8e58eb3d47 100644 --- a/test/framework/tinkerbell.go +++ b/test/framework/tinkerbell.go @@ -28,6 +28,10 @@ var requiredTinkerbellEnvVars = []string{ tinkerbellSSHAuthorizedKey, } +func RequiredTinkerbellEnvVars() []string { + return requiredTinkerbellEnvVars +} + type TinkerbellOpt func(*Tinkerbell) type Tinkerbell struct { From e7af40f0ef86d1ba8768de9d556e0761c15a3754 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Fri, 13 May 2022 16:50:49 -0400 Subject: [PATCH 02/27] split tink tests into instance suites based on hardware capacity --- internal/pkg/api/hardware.go | 18 ++++ internal/pkg/vsphere/deploy.go | 18 ++++ internal/test/e2e/run.go | 153 ++++++++++++++++++++++++-------- internal/test/e2e/setup.go | 5 +- internal/test/e2e/testRunner.go | 31 ++++++- internal/test/e2e/tinkerbell.go | 38 +++++++- test/e2e/forceflow_test.go | 8 +- test/e2e/simpleflow_test.go | 8 +- 8 files changed, 228 insertions(+), 51 deletions(-) diff --git a/internal/pkg/api/hardware.go b/internal/pkg/api/hardware.go index 9847860d9878..e86046ee99f4 100644 --- a/internal/pkg/api/hardware.go +++ b/internal/pkg/api/hardware.go @@ -93,3 +93,21 @@ func WriteHardwareMapToCSV(hardware map[string]*Hardware, csvFile string) error slice := HardwareMapToSlice(hardware) return WriteHardwareSliceToCSV(slice, csvFile) } + +func SplitHardware(slice []*Hardware, chunkSize int) [][]*Hardware { + var chunks [][]*Hardware + for i := 0; i < len(slice); i += chunkSize { + end := i + chunkSize + + // check slice capacity + if end > len(slice) { + end = len(slice) + finalChunk := append(chunks[len(chunks)-1], slice[i:end]...) + chunks[len(chunks)-1] = finalChunk + } else { + chunks = append(chunks, slice[i:end]) + } + } + + return chunks +} diff --git a/internal/pkg/vsphere/deploy.go b/internal/pkg/vsphere/deploy.go index a7e652c4d516..cd2f57fb8446 100644 --- a/internal/pkg/vsphere/deploy.go +++ b/internal/pkg/vsphere/deploy.go @@ -54,3 +54,21 @@ func DeployTemplate(library, templateName, vmName, deployFolder, datacenter, dat return nil } + +func TagVirtualMachine(vmPath, tag string) error { + context := context.Background() + executableBuilder, close, err := executables.NewExecutableBuilder(context, executables.DefaultEksaImage()) + if err != nil { + return fmt.Errorf("unable to initialize executables: %v", err) + } + + defer close.CheckErr(context) + tmpWriter, _ := filewriter.NewWriter(vmPath) + govc := executableBuilder.BuildGovcExecutable(tmpWriter) + defer govc.Close(context) + + if err := govc.AddTag(context, vmPath, tag); err != nil { + return fmt.Errorf("failed to tag vm: %v", err) + } + return nil +} diff --git a/internal/test/e2e/run.go b/internal/test/e2e/run.go index abfdfd588050..f9b5c4e5684f 100644 --- a/internal/test/e2e/run.go +++ b/internal/test/e2e/run.go @@ -4,11 +4,13 @@ import ( "fmt" "os" "regexp" + "strconv" "strings" "sync" "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/eks-anywhere/internal/pkg/ec2" + "github.com/aws/eks-anywhere/internal/pkg/api" + "github.com/aws/eks-anywhere/internal/pkg/s3" "github.com/aws/eks-anywhere/internal/pkg/ssm" "github.com/aws/eks-anywhere/pkg/logger" "github.com/aws/eks-anywhere/pkg/networkutils" @@ -102,7 +104,7 @@ func RunTestsInParallel(conf ParallelRunConf) error { logResult(r.testCommandResult) failedInstances += 1 } else { - logger.Info("Ec2 instance tests completed successfully", "jobId", r.conf.jobId, "instanceId", r.conf.instanceId, "commandId", r.testCommandResult.CommandId, "tests", r.conf.regex, "status", passedStatus) + logger.Info("Instance tests completed successfully", "jobId", r.conf.jobId, "instanceId", r.conf.instanceId, "commandId", r.testCommandResult.CommandId, "tests", r.conf.regex, "status", passedStatus) logResult(r.testCommandResult) } } @@ -119,6 +121,7 @@ type instanceRunConf struct { instanceProfileName, storageBucket, jobId, parentJobId, regex, instanceId string testReportFolder, branchName string ipPool networkutils.IPPool + hardware []*api.Hardware bundlesOverride bool testRunnerType TestRunnerType testRunnerConfig TestInfraConfig @@ -151,6 +154,13 @@ func RunTests(conf instanceRunConf) (testInstanceID string, testCommandResult *t return session.instanceId, nil, err } + key := "Integration-Test-Done" + value := "TRUE" + err = testRunner.tagInstance(conf, key, value) + if err != nil { + return session.instanceId, nil, fmt.Errorf("tagging instance for e2e success: %v", err) + } + return session.instanceId, testCommandResult, nil } @@ -192,13 +202,6 @@ func (c instanceRunConf) runPostTestsProcessing(e *E2ESession, testCommandResult return nil } - key := "Integration-Test-Done" - value := "TRUE" - err := ec2.TagInstance(e.session, e.instanceId, key, value) - if err != nil { - return fmt.Errorf("tagging instance for e2e success: %v", err) - } - return nil } @@ -227,10 +230,25 @@ func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, er runConfs := make([]instanceRunConf, 0, conf.MaxInstances) ipman := newE2EIPManager(os.Getenv(cidrVar), os.Getenv(privateNetworkCidrVar)) - testsInCurrentInstance := make([]string, 0, testPerInstance) + awsSession, err := session.NewSession() + if err != nil { + return nil, fmt.Errorf("creating aws session for test: %v", err) + } + + testRunnerConfig, err := NewTestRunnerConfigFromFile(conf.TestInstanceConfigFile) + if err != nil { + return nil, fmt.Errorf("creating test runner config for tests: %v", err) + } + + testsInEC2Instance := make([]string, 0, testPerInstance) for i, testName := range testsList { - testsInCurrentInstance = append(testsInCurrentInstance, testName) + if tinkerbellTestsRe.MatchString(testName) { + continue + } + + testsInEC2Instance = append(testsInEC2Instance, testName) multiClusterTest := multiClusterTestsRe.MatchString(testName) + var ips networkutils.IPPool if privateNetworkTestsRe.MatchString(testName) { if multiClusterTest { @@ -246,45 +264,102 @@ func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, er } } - awsSession, err := session.NewSession() - if err != nil { - return nil, fmt.Errorf("creating aws session for test: %s, %v", testName, err) + if len(testsInEC2Instance) == testPerInstance || (len(testsList)-1) == i { + runConfs = append(runConfs, newInstanceRunConf(awsSession, conf, len(runConfs), strings.Join(testsInEC2Instance, "|"), ips, []*api.Hardware{}, Ec2TestRunnerType, testRunnerConfig)) + testsInEC2Instance = make([]string, 0, testPerInstance) } + } - testRunnerConfig, err := NewTestRunnerConfigFromFile(conf.TestInstanceConfigFile) - if err != nil { - return nil, fmt.Errorf("creating test runner config for test: %s, %v", testName, err) - } + err = s3.DownloadToDisk(awsSession, os.Getenv(tinkerbellHardwareS3FileKeyEnvVar), conf.StorageBucket, e2eHardwareCsvFilePath) + if err != nil { + return nil, fmt.Errorf("failed to download tinkerbell hardware csv: %v", err) + } - testRunnerType := Ec2TestRunnerType - if tinkerbellTestsRe.MatchString(testName) { - testRunnerType = VSphereTestRunnerType + hardware, err := api.NewHardwareSliceFromFile(e2eHardwareCsvFilePath) + if err != nil { + return nil, fmt.Errorf("failed to get Tinkerbell hardware: %v", err) + } + + maxHardwarePerE2ETest, err := strconv.Atoi(os.Getenv(MaxHardwarePerE2ETestEnvVar)) + if err != nil { + return nil, fmt.Errorf("failed to get Tinkerbell max hardware per test env var: %v", err) + } + + logger.V(1).Info("INFO:", "totalHardware", len(hardware)) + + tinkerbellTests := getTinkerbellTests(testsList) + logger.V(1).Info("INFO:", "tinkerbellTests", len(tinkerbellTests)) + + tinkTestInstances := len(hardware) / maxHardwarePerE2ETest + logger.V(1).Info("INFO:", "tinkTestInstances", tinkTestInstances) + + tinkTestsPerInstance := 1 + var remainingTests int + overflowTests := false + if len(tinkerbellTests) > tinkTestInstances { + tinkTestsPerInstance = len(tinkerbellTests) / tinkTestInstances + remainingTests = len(tinkerbellTests) % tinkTestInstances + if remainingTests != 0 { + tinkTestsPerInstance++ + overflowTests = true } + } + + logger.V(1).Info("INFO:", "tinkTestsPerInstance", tinkTestsPerInstance) + logger.V(1).Info("INFO:", "tinkTestInstances", tinkTestInstances) + logger.V(1).Info("INFO:", "remainingTests", remainingTests) + + hardwareChunks := api.SplitHardware(hardware, maxHardwarePerE2ETest) - if len(testsInCurrentInstance) == testPerInstance || (len(testsList)-1) == i { - runConfs = append(runConfs, instanceRunConf{ - session: awsSession, - instanceProfileName: conf.InstanceProfileName, - storageBucket: conf.StorageBucket, - jobId: fmt.Sprintf("%s-%d", conf.JobId, len(runConfs)), - parentJobId: conf.JobId, - regex: strings.Join(testsInCurrentInstance, "|"), - bundlesOverride: conf.BundlesOverride, - cleanupVms: conf.CleanupVms, - testReportFolder: conf.TestReportFolder, - branchName: conf.BranchName, - ipPool: ips, - testRunnerType: testRunnerType, - testRunnerConfig: *testRunnerConfig, - }) - - testsInCurrentInstance = make([]string, 0, testPerInstance) + testsInVSphereInstance := make([]string, 0, tinkTestsPerInstance) + for i, testName := range tinkerbellTests { + testsInVSphereInstance = append(testsInVSphereInstance, testName) + + if len(testsInVSphereInstance) == tinkTestsPerInstance || (len(testsList)-1) == i { + logger.V(1).Info("INFO:", "hardwareChunksSize", len(hardwareChunks)) + logger.V(1).Info("INFO:", "hardwareSize", len(hardware)) + + if len(hardwareChunks) > 0 { + hardware, hardwareChunks = hardwareChunks[0], hardwareChunks[1:] + } + + runConfs = append(runConfs, newInstanceRunConf(awsSession, conf, len(runConfs), strings.Join(testsInVSphereInstance, "|"), networkutils.IPPool{}, hardware, VSphereTestRunnerType, testRunnerConfig)) + + if remainingTests > 0 { + remainingTests-- + } + + if remainingTests == 0 && overflowTests { + tinkTestsPerInstance-- + overflowTests = false + } + + testsInVSphereInstance = make([]string, 0, tinkTestsPerInstance) } } return runConfs, nil } +func newInstanceRunConf(awsSession *session.Session, conf ParallelRunConf, jobNumber int, testRegex string, ipPool networkutils.IPPool, hardware []*api.Hardware, testRunnerType TestRunnerType, testRunnerConfig *TestInfraConfig) instanceRunConf { + return instanceRunConf{ + session: awsSession, + instanceProfileName: conf.InstanceProfileName, + storageBucket: conf.StorageBucket, + jobId: fmt.Sprintf("%s-%d", conf.JobId, jobNumber), + parentJobId: conf.JobId, + regex: testRegex, + ipPool: ipPool, + hardware: hardware, + bundlesOverride: conf.BundlesOverride, + testReportFolder: conf.TestReportFolder, + branchName: conf.BranchName, + cleanupVms: conf.CleanupVms, + testRunnerType: testRunnerType, + testRunnerConfig: *testRunnerConfig, + } +} + func logTestGroups(instancesConf []instanceRunConf) { testGroups := make([]string, 0, len(instancesConf)) for _, i := range instancesConf { diff --git a/internal/test/e2e/setup.go b/internal/test/e2e/setup.go index 82e15d0c3f90..cc12abcae6c1 100644 --- a/internal/test/e2e/setup.go +++ b/internal/test/e2e/setup.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/eks-anywhere/internal/pkg/api" "github.com/aws/eks-anywhere/internal/pkg/s3" "github.com/aws/eks-anywhere/internal/pkg/ssm" "github.com/aws/eks-anywhere/pkg/logger" @@ -42,6 +43,7 @@ type E2ESession struct { cleanupVms bool requiredFiles []string branchName string + hardware []*api.Hardware } func newE2ESession(instanceId string, conf instanceRunConf) (*E2ESession, error) { @@ -57,6 +59,7 @@ func newE2ESession(instanceId string, conf instanceRunConf) (*E2ESession, error) cleanupVms: conf.cleanupVms, requiredFiles: requiredFiles, branchName: conf.branchName, + hardware: conf.hardware, } return e, nil @@ -214,7 +217,7 @@ func (e *E2ESession) downloadRequiredFilesInInstance() error { } func (e *E2ESession) createTestNameFile(testName string) error { - command := fmt.Sprintf("echo %s > %s", testName, testNameFile) + command := fmt.Sprintf("echo \"%s\" > %s", testName, testNameFile) if err := ssm.Run(e.session, e.instanceId, command); err != nil { return fmt.Errorf("creating test name file in instance: %v", err) diff --git a/internal/test/e2e/testRunner.go b/internal/test/e2e/testRunner.go index c8df89a2196c..60629ae1c536 100644 --- a/internal/test/e2e/testRunner.go +++ b/internal/test/e2e/testRunner.go @@ -29,6 +29,7 @@ const ( type TestRunner interface { createInstance(instanceConf instanceRunConf) (string, error) + tagInstance(instanceConf instanceRunConf, key, value string) error } type TestRunnerType string @@ -67,12 +68,18 @@ func NewTestRunnerConfigFromFile(configFile string) (*TestInfraConfig, error) { return &config, nil } +type testRunner struct { + InstanceID string +} + type Ec2TestRunner struct { + testRunner AmiID string `yaml:"amiId"` SubnetID string `yaml:"subnetId"` } type VSphereTestRunner struct { + testRunner Url string `yaml:"url"` Insecure bool `yaml:"insecure"` Library string `yaml:"library"` @@ -126,7 +133,8 @@ func (v VSphereTestRunner) createInstance(c instanceRunConf) (string, error) { return "", fmt.Errorf("unable to create ssm activation: %v", err) } - // import ova template from url if not exist + // TODO: import ova template from url if not exist + opts := vsphere.OVFDeployOptions{ Name: name, PowerOn: true, @@ -157,7 +165,7 @@ func (v VSphereTestRunner) createInstance(c instanceRunConf) (string, error) { if err != nil { return "", fmt.Errorf("waiting for ssm instance to activate %s : %v", name, err) } - + v.InstanceID = *ssmInstance.InstanceId return *ssmInstance.InstanceId, nil } @@ -169,9 +177,28 @@ func (e Ec2TestRunner) createInstance(c instanceRunConf) (string, error) { return "", fmt.Errorf("creating instance for e2e tests: %v", err) } logger.V(1).Info("Instance created", "instance-id", instanceId) + e.InstanceID = instanceId return instanceId, nil } +func (v VSphereTestRunner) tagInstance(c instanceRunConf, key, value string) error { + vmName := getTestRunnerName(c.jobId) + vmPath := fmt.Sprintf("/%s/vm/%s/%s", v.Datacenter, v.Folder, vmName) + tag := fmt.Sprintf("%s:%s", key, value) + if err := vsphere.TagVirtualMachine(vmPath, tag); err != nil { + return fmt.Errorf("failed to tag vSphere test runner: %v", err) + } + return nil +} + +func (e Ec2TestRunner) tagInstance(c instanceRunConf, key, value string) error { + err := ec2.TagInstance(c.session, c.instanceId, key, value) + if err != nil { + return fmt.Errorf("failed to tag Ec2 test runner: %v", err) + } + return nil +} + func getTestRunnerName(jobId string) string { return fmt.Sprintf("eksa-e2e-%s", jobId) } diff --git a/internal/test/e2e/tinkerbell.go b/internal/test/e2e/tinkerbell.go index ea869497d6cd..05eb983b28c3 100644 --- a/internal/test/e2e/tinkerbell.go +++ b/internal/test/e2e/tinkerbell.go @@ -1,15 +1,21 @@ package e2e import ( + "fmt" "os" "regexp" + "github.com/aws/eks-anywhere/internal/pkg/api" "github.com/aws/eks-anywhere/pkg/logger" e2etests "github.com/aws/eks-anywhere/test/framework" ) const ( - tinkerbellTestsRe = `^.*Tinkerbell.*$` + tinkerbellInventoryCsvFilePathEnvVar = "T_TINKERBELL_INVENTORY_CSV" + tinkerbellHardwareS3FileKeyEnvVar = "T_TINKERBELL_S3_INVENTORY_CSV_KEY" + tinkerbellTestsRe = `^.*Tinkerbell.*$` + e2eHardwareCsvFilePath = "e2e-inventory.csv" + MaxHardwarePerE2ETestEnvVar = "T_TINKERBELL_MAX_HARDWARE_PER_TEST" ) func (e *E2ESession) setupTinkerbellEnv(testRegex string) error { @@ -26,5 +32,35 @@ func (e *E2ESession) setupTinkerbellEnv(testRegex string) error { } } + inventoryFileName := fmt.Sprintf("%s.csv", getTestRunnerName(e.jobId)) + inventoryFilePath := fmt.Sprintf("bin/%s", inventoryFileName) + err := api.WriteHardwareSliceToCSV(e.hardware, inventoryFilePath) + if err != nil { + return fmt.Errorf("failed to setup tinkerbell test environment: %v", err) + } + + err = e.uploadRequiredFile(inventoryFileName) + if err != nil { + return fmt.Errorf("failed to upload tinkerbell inventory file (%s) : %v", inventoryFileName, err) + } + + err = e.downloadRequiredFileInInstance(inventoryFileName) + if err != nil { + return fmt.Errorf("failed to download tinkerbell inventory file (%s) to test instance : %v", inventoryFileName, err) + } + + e.testEnvVars[tinkerbellInventoryCsvFilePathEnvVar] = inventoryFilePath + return nil } + +func getTinkerbellTests(tests []string) []string { + tinkerbellTestsRe := regexp.MustCompile(tinkerbellTestsRe) + var tinkerbellTests []string + for _, testName := range tests { + if tinkerbellTestsRe.MatchString(testName) { + tinkerbellTests = append(tinkerbellTests, testName) + } + } + return tinkerbellTests +} diff --git a/test/e2e/forceflow_test.go b/test/e2e/forceflow_test.go index c07b39d61767..13824fb79624 100644 --- a/test/e2e/forceflow_test.go +++ b/test/e2e/forceflow_test.go @@ -14,10 +14,10 @@ import ( func runTinkerbellForceFlow(test *framework.ClusterE2ETest) { test.GenerateClusterConfig() test.GenerateHardwareConfig() - test.PowerOnHardware() - test.CreateCluster(framework.WithForce()) - test.DeleteCluster() - test.ValidateHardwareDecommissioned() + //test.PowerOnHardware() + //test.CreateCluster(framework.WithForce()) + //test.DeleteCluster() + //test.ValidateHardwareDecommissioned() } func TestTinkerbellKubernetes121ForceFlow(t *testing.T) { diff --git a/test/e2e/simpleflow_test.go b/test/e2e/simpleflow_test.go index ada04ae39bb7..d56ca1aa656d 100644 --- a/test/e2e/simpleflow_test.go +++ b/test/e2e/simpleflow_test.go @@ -31,10 +31,10 @@ func runHelmInstallSimpleFlow(test *framework.ClusterE2ETest) { func runTinkerbellSimpleFlow(test *framework.ClusterE2ETest) { test.GenerateClusterConfig() test.GenerateHardwareConfig() - test.PowerOffHardware() - test.CreateCluster() - test.DeleteCluster() - test.ValidateHardwareDecommissioned() + //test.PowerOffHardware() + //test.CreateCluster() + //test.DeleteCluster() + //test.ValidateHardwareDecommissioned() } func runSimpleFlow(test *framework.ClusterE2ETest) { From ff0dc3691d073fa384ad80f2683d0e273d3a543d Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Wed, 1 Jun 2022 17:15:48 -0400 Subject: [PATCH 03/27] account for multiple tests in single instance decomm test runner --- internal/pkg/ssm/activation.go | 15 ++++++++++++ internal/pkg/ssm/instance.go | 14 +++++++++++ internal/test/e2e/artifacts.go | 4 ++-- internal/test/e2e/run.go | 31 ++++++++++++++++-------- internal/test/e2e/setup.go | 12 ++++++---- internal/test/e2e/testRunner.go | 42 ++++++++++++++++++++++++++++----- test/framework/cluster.go | 27 ++++++++++++--------- 7 files changed, 112 insertions(+), 33 deletions(-) diff --git a/internal/pkg/ssm/activation.go b/internal/pkg/ssm/activation.go index e3c8954867e6..b92ca48d4707 100644 --- a/internal/pkg/ssm/activation.go +++ b/internal/pkg/ssm/activation.go @@ -28,3 +28,18 @@ func CreateActivation(session *session.Session, instanceName, role string) (*Act return &ActivationInfo{ActivationCode: *result.ActivationCode, ActivationID: *result.ActivationId}, nil } + +func DeleteActivation(session *session.Session, activationId string) (*ssm.DeleteActivationOutput, error) { + s := ssm.New(session) + + request := ssm.DeleteActivationInput{ + ActivationId: &activationId, + } + + result, err := s.DeleteActivation(&request) + if err != nil { + return nil, fmt.Errorf("failed to delete ssm activation: %v", err) + } + + return result, nil +} diff --git a/internal/pkg/ssm/instance.go b/internal/pkg/ssm/instance.go index 5beae2358414..6fd2b4b80ae2 100644 --- a/internal/pkg/ssm/instance.go +++ b/internal/pkg/ssm/instance.go @@ -29,3 +29,17 @@ func GetInstanceByActivationId(session *session.Session, id string) (*ssm.Instan return infoList[0], nil } + +func DeregisterInstance(session *session.Session, id string) (*ssm.DeregisterManagedInstanceOutput, error) { + s := ssm.New(session) + input := ssm.DeregisterManagedInstanceInput{ + InstanceId: &id, + } + + output, err := s.DeregisterManagedInstance(&input) + if err != nil { + return nil, fmt.Errorf("failed to deregister ssm instance %s: %v", id, err) + } + + return output, nil +} diff --git a/internal/test/e2e/artifacts.go b/internal/test/e2e/artifacts.go index 7eb099c43094..22eafdbf11a5 100644 --- a/internal/test/e2e/artifacts.go +++ b/internal/test/e2e/artifacts.go @@ -14,7 +14,7 @@ const e2eHomeFolder = "/home/e2e/" func (e *E2ESession) uploadGeneratedFilesFromInstance(testName string) { logger.V(1).Info("Uploading log files to s3 bucket") command := newCopyCommand().from( - e2eHomeFolder, clusterName(e.branchName, e.instanceId), + e2eHomeFolder, clusterName(e.branchName, e.instanceId, testName), ).to( e.generatedArtifactsBucketPath(), testName, ).recursive().String() @@ -29,7 +29,7 @@ func (e *E2ESession) uploadGeneratedFilesFromInstance(testName string) { func (e *E2ESession) uploadDiagnosticArchiveFromInstance(testName string) { bundleNameFormat := "support-bundle-*.tar.gz" logger.V(1).Info("Uploading diagnostic bundle to s3 bucket") - command := newCopyCommand().from(e2eHomeFolder).to( + command := newCopyCommand().from(e2eHomeFolder, clusterName(e.branchName, e.instanceId, testName)).to( e.generatedArtifactsBucketPath(), testName, ).recursive().exclude("*").include(bundleNameFormat).String() diff --git a/internal/test/e2e/run.go b/internal/test/e2e/run.go index f9b5c4e5684f..42b441ab5af5 100644 --- a/internal/test/e2e/run.go +++ b/internal/test/e2e/run.go @@ -135,6 +135,13 @@ func RunTests(conf instanceRunConf) (testInstanceID string, testCommandResult *t return "", nil, err } + defer func() { + err := testRunner.decommInstance(conf) + if err != nil { + logger.V(1).Info("WARN: Failed to decomm e2e test runner instance", "error", err) + } + }() + session, err := newE2ESession(instanceId, conf) if err != nil { return "", nil, err @@ -169,7 +176,7 @@ func (e *E2ESession) runTests(regex string) (testCommandResult *testCommandResul command := "GOVERSION=go1.16.6 gotestsum --junitfile=junit-testing.xml --raw-command --format=standard-verbose --hide-summary=all --ignore-non-json-output-lines -- test2json -t -p e2e ./bin/e2e.test -test.v" if regex != "" { - command = fmt.Sprintf("%s -test.run %s", command, regex) + command = fmt.Sprintf("%s -test.run \"%s\"", command, regex) } command = e.commandWithEnvVars(command) @@ -190,16 +197,20 @@ func (e *E2ESession) runTests(regex string) (testCommandResult *testCommandResul } func (c instanceRunConf) runPostTestsProcessing(e *E2ESession, testCommandResult *testCommandResult) error { - testName := strings.Trim(c.regex, "\"") - e.uploadJUnitReportFromInstance(testName) - if c.testReportFolder != "" { - e.downloadJUnitReportToLocalDisk(testName, c.testReportFolder) - } + regex := strings.Trim(c.regex, "\"") + tests := strings.Split(regex, "|") - if !testCommandResult.Successful() { - e.uploadGeneratedFilesFromInstance(testName) - e.uploadDiagnosticArchiveFromInstance(testName) - return nil + for _, testName := range tests { + e.uploadJUnitReportFromInstance(testName) + if c.testReportFolder != "" { + e.downloadJUnitReportToLocalDisk(testName, c.testReportFolder) + } + + if !testCommandResult.Successful() { + e.uploadGeneratedFilesFromInstance(testName) + e.uploadDiagnosticArchiveFromInstance(testName) + return nil + } } return nil diff --git a/internal/test/e2e/setup.go b/internal/test/e2e/setup.go index cc12abcae6c1..5752e21a90bd 100644 --- a/internal/test/e2e/setup.go +++ b/internal/test/e2e/setup.go @@ -145,7 +145,7 @@ func (e *E2ESession) setup(regex string) error { e.testEnvVars[e2etests.BranchNameEnvVar] = e.branchName } - e.testEnvVars[e2etests.ClusterNameVar] = clusterName(e.branchName, e.instanceId) + e.testEnvVars[e2etests.ClusterPrefixVar] = clusterPrefix(e.branchName, e.instanceId) return nil } @@ -227,17 +227,21 @@ func (e *E2ESession) createTestNameFile(testName string) error { return nil } -func clusterName(branch string, instanceId string) (clusterName string) { +func clusterPrefix(branch, instanceId string) (clusterPrefix string) { if branch == "" { return instanceId } - clusterNameTemplate := "%s-%s" forbiddenChars := []string{"."} sanitizedBranch := strings.ToLower(branch) for _, char := range forbiddenChars { sanitizedBranch = strings.ReplaceAll(sanitizedBranch, char, "-") } - clusterName = fmt.Sprintf(clusterNameTemplate, sanitizedBranch, instanceId) + clusterPrefix = fmt.Sprintf("%s-%s", sanitizedBranch, instanceId) + return clusterPrefix +} + +func clusterName(branch, instanceId, testName string) (clusterName string) { + clusterName = fmt.Sprintf("%s-%s", clusterPrefix(branch, instanceId), e2etests.GetTestNameHash(testName)) if len(clusterName) > 80 { logger.Info("Cluster name is longer than 80 characters; truncating to 80 characters.", "original cluster name", clusterName, "truncated cluster name", clusterName[:80]) clusterName = clusterName[:80] diff --git a/internal/test/e2e/testRunner.go b/internal/test/e2e/testRunner.go index 60629ae1c536..6b1c406ce075 100644 --- a/internal/test/e2e/testRunner.go +++ b/internal/test/e2e/testRunner.go @@ -1,6 +1,7 @@ package e2e import ( + "context" "fmt" "os" "strconv" @@ -30,6 +31,7 @@ const ( type TestRunner interface { createInstance(instanceConf instanceRunConf) (string, error) tagInstance(instanceConf instanceRunConf, key, value string) error + decommInstance(instanceRunConf) error } type TestRunnerType string @@ -41,9 +43,9 @@ const ( func newTestRunner(runnerType TestRunnerType, config TestInfraConfig) TestRunner { if runnerType == VSphereTestRunnerType { - return config.VSphereTestRunner + return &config.VSphereTestRunner } else { - return config.Ec2TestRunner + return &config.Ec2TestRunner } } @@ -80,6 +82,7 @@ type Ec2TestRunner struct { type VSphereTestRunner struct { testRunner + ActivationId string Url string `yaml:"url"` Insecure bool `yaml:"insecure"` Library string `yaml:"library"` @@ -119,7 +122,7 @@ func (v *VSphereTestRunner) setEnvironment() error { return nil } -func (v VSphereTestRunner) createInstance(c instanceRunConf) (string, error) { +func (v *VSphereTestRunner) createInstance(c instanceRunConf) (string, error) { name := getTestRunnerName(c.jobId) logger.V(1).Info("Creating vSphere Test Runner instance", "name", name) @@ -165,11 +168,14 @@ func (v VSphereTestRunner) createInstance(c instanceRunConf) (string, error) { if err != nil { return "", fmt.Errorf("waiting for ssm instance to activate %s : %v", name, err) } + v.InstanceID = *ssmInstance.InstanceId + v.ActivationId = ssmActivationInfo.ActivationID + return *ssmInstance.InstanceId, nil } -func (e Ec2TestRunner) createInstance(c instanceRunConf) (string, error) { +func (e *Ec2TestRunner) createInstance(c instanceRunConf) (string, error) { name := getTestRunnerName(c.jobId) logger.V(1).Info("Creating ec2 Test Runner instance", "name", name) instanceId, err := ec2.CreateInstance(c.session, e.AmiID, key, tag, c.instanceProfileName, e.SubnetID, name) @@ -181,7 +187,7 @@ func (e Ec2TestRunner) createInstance(c instanceRunConf) (string, error) { return instanceId, nil } -func (v VSphereTestRunner) tagInstance(c instanceRunConf, key, value string) error { +func (v *VSphereTestRunner) tagInstance(c instanceRunConf, key, value string) error { vmName := getTestRunnerName(c.jobId) vmPath := fmt.Sprintf("/%s/vm/%s/%s", v.Datacenter, v.Folder, vmName) tag := fmt.Sprintf("%s:%s", key, value) @@ -191,7 +197,7 @@ func (v VSphereTestRunner) tagInstance(c instanceRunConf, key, value string) err return nil } -func (e Ec2TestRunner) tagInstance(c instanceRunConf, key, value string) error { +func (e *Ec2TestRunner) tagInstance(c instanceRunConf, key, value string) error { err := ec2.TagInstance(c.session, c.instanceId, key, value) if err != nil { return fmt.Errorf("failed to tag Ec2 test runner: %v", err) @@ -199,6 +205,30 @@ func (e Ec2TestRunner) tagInstance(c instanceRunConf, key, value string) error { return nil } +func (v *VSphereTestRunner) decommInstance(c instanceRunConf) error { + _, deregisterError := ssm.DeregisterInstance(c.session, v.InstanceID) + _, deactivateError := ssm.DeleteActivation(c.session, v.ActivationId) + deleteError := vsphereRmVms(context.Background(), getTestRunnerName(c.jobId)) + + if deregisterError != nil { + return fmt.Errorf("failed to decommission vsphere test runner ssm instance: %v", deregisterError) + } + + if deactivateError != nil { + return fmt.Errorf("failed to decommission vsphere test runner ssm instance: %v", deactivateError) + } + + if deleteError != nil { + return fmt.Errorf("failed to decommission vsphere test runner ssm instance: %v", deleteError) + } + + return nil +} + +func (v *Ec2TestRunner) decommInstance(c instanceRunConf) error { + return nil +} + func getTestRunnerName(jobId string) string { return fmt.Sprintf("eksa-e2e-%s", jobId) } diff --git a/test/framework/cluster.go b/test/framework/cluster.go index 815015dddfb1..3d6aea45aab7 100644 --- a/test/framework/cluster.go +++ b/test/framework/cluster.go @@ -36,7 +36,7 @@ const ( defaultClusterName = "eksa-test" eksctlVersionEnvVar = "EKSCTL_VERSION" eksctlVersionEnvVarDummyVal = "ham sandwich" - ClusterNameVar = "T_CLUSTER_NAME" + ClusterPrefixVar = "T_CLUSTER_PREFIX" JobIdVar = "T_JOB_ID" BundlesOverrideVar = "T_BUNDLES_OVERRIDE" CleanupVmsVar = "T_CLEANUP_VMS" @@ -87,7 +87,7 @@ func NewClusterE2ETest(t *testing.T, provider Provider, opts ...ClusterE2ETestOp eksaBinaryLocation: defaultEksaBinaryLocation, } - e.ClusterConfigFolder = fmt.Sprintf("%s-config", e.ClusterName) + e.ClusterConfigFolder = e.ClusterName e.HardwareConfigLocation = filepath.Join(e.ClusterConfigFolder, hardwareYamlPath) e.HardwareCsvLocation = filepath.Join(e.ClusterConfigFolder, hardwareCsvPath) @@ -604,17 +604,22 @@ func (e *ClusterE2ETest) getJobIdFromEnv() string { return os.Getenv(JobIdVar) } +func GetTestNameHash(name string) string { + h := sha1.New() + h.Write([]byte(name)) + testNameHash := fmt.Sprintf("%x", h.Sum(nil)) + return testNameHash[:7] +} + func getClusterName(t *testing.T) string { - value := os.Getenv(ClusterNameVar) + value := os.Getenv(ClusterPrefixVar) + // Append hash to make each cluster name unique per test. Using the testname will be too long + // and would fail validations if len(value) == 0 { - h := sha1.New() - h.Write([]byte(t.Name())) - testNameHash := fmt.Sprintf("%x", h.Sum(nil)) - // Append hash to make each cluster name unique per test. Using the testname will be too long - // and would fail validations - return fmt.Sprintf("%s-%s", defaultClusterName, testNameHash[:7]) - } - return value + value = defaultClusterName + } + + return fmt.Sprintf("%s-%s", value, GetTestNameHash(t.Name())) } func getBundlesOverride() string { From dd1fcc47662485e43553fd7138b4aad63eb8743b Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 2 Jun 2022 19:59:33 -0400 Subject: [PATCH 04/27] fix generate hardware & marshall labels --- cmd/eksctl-anywhere/cmd/generatehardware.go | 2 +- .../build/buildspecs/test-infra.yml | 14 +- go.mod | 11 +- go.sum | 9 ++ internal/test/e2e/tinkerbell.go | 28 ++++ pkg/providers/tinkerbell/hardware/machine.go | 6 +- pkg/providers/tinkerbell/hardware/yaml.go | 6 + test/e2e/forceflow_test.go | 56 -------- test/e2e/simpleflow_test.go | 55 ++------ test/framework/cluster.go | 128 ++++++++++++++++-- test/framework/tinkerbell.go | 13 +- 11 files changed, 207 insertions(+), 121 deletions(-) delete mode 100644 test/e2e/forceflow_test.go diff --git a/cmd/eksctl-anywhere/cmd/generatehardware.go b/cmd/eksctl-anywhere/cmd/generatehardware.go index 61124fab08f4..741f1218d45e 100644 --- a/cmd/eksctl-anywhere/cmd/generatehardware.go +++ b/cmd/eksctl-anywhere/cmd/generatehardware.go @@ -32,7 +32,7 @@ func init() { flags := generateHardwareCmd.Flags() flags.StringVarP(&hOpts.outputPath, "output", "o", "", "Path to output hardware YAML.") flags.StringVarP( - &cc.hardwareCSVPath, + &hOpts.csvPath, TinkerbellHardwareCSVFlagName, TinkerbellHardwareCSVFlagAlias, "", diff --git a/cmd/integration_test/build/buildspecs/test-infra.yml b/cmd/integration_test/build/buildspecs/test-infra.yml index 2b6b30bc5b3f..5be622230e9d 100644 --- a/cmd/integration_test/build/buildspecs/test-infra.yml +++ b/cmd/integration_test/build/buildspecs/test-infra.yml @@ -5,12 +5,12 @@ ec2: subnetId: subnet-0fd4b6ba7c9e17b5f vSphere: - url: https://10.61.250.74 + url: https://vcenter.sddc-44-239-186-141.vmwarevmc.com insecure: True library: eks-a-templates - template: eks-a-admin-v0.0.0-8a498c1-20220511153117 - datacenter: Datacenter - datastore: datastore2 - resourcePool: TestResourcePool - network: VM Network - folder: gwesterf \ No newline at end of file + template: eks-a-admin-ipmitool + datacenter: SDDC-Datacenter + datastore: WorkloadDatastore + resourcePool: Compute-ResourcePool + network: sddc-cgw-network-4 + folder: gwest \ No newline at end of file diff --git a/go.mod b/go.mod index ccb4e0398710..ce9e5292b378 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( golang.org/x/sys v0.0.0-20220412211240-33da011f77ad gopkg.in/ini.v1 v1.66.2 gopkg.in/square/go-jose.v2 v2.6.0 + gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b k8s.io/api v0.23.4 k8s.io/apimachinery v0.23.4 @@ -70,6 +71,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect + github.com/bmc-toolbox/bmclib v0.5.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/containerd/containerd v1.6.1 // indirect github.com/coredns/caddy v1.1.0 // indirect @@ -88,6 +90,8 @@ require ( github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect github.com/gobuffalo/flect v0.2.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.0.0 // indirect @@ -98,16 +102,20 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/gorilla/mux v1.8.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/itchyny/gojq v0.12.6 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect + github.com/jacobweinstock/registrar v0.4.6 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.1.0 // indirect github.com/klauspost/compress v1.13.6 // indirect + github.com/leodido/go-urn v1.2.1 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/matryer/is v1.4.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect @@ -132,6 +140,7 @@ require ( github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/stmcginnis/gofish v0.12.1-0.20220311113027-6072260f4c8d // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect go.uber.org/atomic v1.9.0 // indirect @@ -145,9 +154,9 @@ require ( google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect google.golang.org/grpc v1.43.0 // indirect google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/go-playground/validator.v9 v9.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/apiextensions-apiserver v0.23.4 // indirect k8s.io/cluster-bootstrap v0.23.0 // indirect k8s.io/component-base v0.23.4 // indirect diff --git a/go.sum b/go.sum index aae754a2a6dd..0bce455c1106 100644 --- a/go.sum +++ b/go.sum @@ -316,8 +316,10 @@ github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdn github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= +github.com/bmc-toolbox/bmclib v0.5.3 h1:aGYvGwEUZOIRP1mhhzELVPDGWsES7CsDLANciJ1TgF4= github.com/bmc-toolbox/bmclib v0.5.3/go.mod h1:GR1j74vPi8vlos514fwtg0dWq1khieqDTsArL5ae2jk= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= +github.com/bombsimon/logrusr/v2 v2.0.1 h1:1VgxVNQMCvjirZIYaT9JYn6sAVGVEcNtRE0y4mvaOAM= github.com/bombsimon/logrusr/v2 v2.0.1/go.mod h1:ByVAX+vHdLGAfdroiMg6q0zgq2FODY2lc5YJvzmOJio= github.com/bombsimon/wsl/v3 v3.3.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= @@ -684,7 +686,9 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+ github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= @@ -1026,6 +1030,7 @@ github.com/itchyny/gojq v0.12.6/go.mod h1:ZHrkfu7A+RbZLy5J1/JKpS4poEqrzItSTGDItq github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jacobweinstock/registrar v0.4.6 h1:0O3g2jT2Lx+Bf+yl4QsMUN48fVZxUpM3kS+NtIJ+ucw= github.com/jacobweinstock/registrar v0.4.6/go.mod h1:IDx65tQ7DLJ2UqiVjE1zo74jMZZfel9YZW8VrC26m6o= github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -1142,6 +1147,7 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm github.com/ldez/gomoddirectives v0.2.1/go.mod h1:sGicqkRgBOg//JfpXwkB9Hj0X5RyJ7mlACM5B9f6Me4= github.com/ldez/tagliatelle v0.2.0/go.mod h1:8s6WJQwEYHbKZDsp/LjArytKOG8qaMrKQQ3mFukHs88= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/letsencrypt/pkcs11key/v4 v4.0.0/go.mod h1:EFUvBDay26dErnNb70Nd0/VW3tJiIbETBPTl9ATXQag= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -1595,6 +1601,7 @@ github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/ssgreg/nlreturn/v2 v2.1.0/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= +github.com/stmcginnis/gofish v0.12.1-0.20220311113027-6072260f4c8d h1:cqKAdqKgyLY3CbHNViBPALcUbbYSXOciB5lppURZ3lE= github.com/stmcginnis/gofish v0.12.1-0.20220311113027-6072260f4c8d/go.mod h1:BLDSFTp8pDlf/xDbLZa+F7f7eW0E/CHCboggsu8CznI= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -2517,7 +2524,9 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= diff --git a/internal/test/e2e/tinkerbell.go b/internal/test/e2e/tinkerbell.go index 05eb983b28c3..7c8836d83cca 100644 --- a/internal/test/e2e/tinkerbell.go +++ b/internal/test/e2e/tinkerbell.go @@ -6,6 +6,7 @@ import ( "regexp" "github.com/aws/eks-anywhere/internal/pkg/api" + "github.com/aws/eks-anywhere/internal/pkg/ssm" "github.com/aws/eks-anywhere/pkg/logger" e2etests "github.com/aws/eks-anywhere/test/framework" ) @@ -34,6 +35,14 @@ func (e *E2ESession) setupTinkerbellEnv(testRegex string) error { inventoryFileName := fmt.Sprintf("%s.csv", getTestRunnerName(e.jobId)) inventoryFilePath := fmt.Sprintf("bin/%s", inventoryFileName) + + if _, err := os.Stat(inventoryFilePath); err == nil { + e := os.Remove(inventoryFilePath) + if e != nil { + logger.V(1).Info("WARN: Failed to clean up existing inventory csv", "file", inventoryFilePath) + } + } + err := api.WriteHardwareSliceToCSV(e.hardware, inventoryFilePath) if err != nil { return fmt.Errorf("failed to setup tinkerbell test environment: %v", err) @@ -49,11 +58,30 @@ func (e *E2ESession) setupTinkerbellEnv(testRegex string) error { return fmt.Errorf("failed to download tinkerbell inventory file (%s) to test instance : %v", inventoryFileName, err) } + tinkInterface := "ens192" + err = e.setTinkerbellBootstrapIPInInstance(tinkInterface) + if err != nil { + return fmt.Errorf("failed to set tinkerbell boostrap ip on interface (%s) in test instance : %v", tinkInterface, err) + } + e.testEnvVars[tinkerbellInventoryCsvFilePathEnvVar] = inventoryFilePath return nil } +func (e *E2ESession) setTinkerbellBootstrapIPInInstance(tinkInterface string) error { + logger.V(1).Info("Setting Tinkerbell Bootstrap IP in instance") + + command := fmt.Sprintf("export TINKERBELL_BOOSTRAP_IP=$(/sbin/ip -o -4 addr list %s | awk '{print $4}' | cut -d/ -f1) && echo TINKERBELL_BOOTSTRAP_IP=$TINKERBELL_BOOSTRAP_IP | tee -a /etc/environment", tinkInterface) + if err := ssm.Run(e.session, e.instanceId, command); err != nil { + return fmt.Errorf("setting tinkerbell boostrap ip: %v", err) + } + + logger.V(1).Info("Successfully set tinkerbell boostrap ip") + + return nil +} + func getTinkerbellTests(tests []string) []string { tinkerbellTestsRe := regexp.MustCompile(tinkerbellTestsRe) var tinkerbellTests []string diff --git a/pkg/providers/tinkerbell/hardware/machine.go b/pkg/providers/tinkerbell/hardware/machine.go index 60106156b74a..e2c36e450b3f 100644 --- a/pkg/providers/tinkerbell/hardware/machine.go +++ b/pkg/providers/tinkerbell/hardware/machine.go @@ -17,7 +17,7 @@ type Machine struct { // Disk used to populate the default workflow actions. // Currently needs to be the same for all hardware residing in the same group where a group - // is either: control plane hardwar, external etcd hard, or the definable worker node groups. + // is either: control plane hardware, external etcd hard, or the definable worker node groups. Disk string `csv:"disk"` // Labels to be applied to the Hardware resource. @@ -73,6 +73,10 @@ func (l Labels) Get(k string) string { return l[k] } +func (l *Labels) MarshalCSV() (string, error) { + return l.String(), nil +} + func (l *Labels) UnmarshalCSV(s string) error { // Ensure we make the map so consumers of l don't segfault. *l = make(Labels) diff --git a/pkg/providers/tinkerbell/hardware/yaml.go b/pkg/providers/tinkerbell/hardware/yaml.go index 4d8d08d350dd..d3608d5dc641 100644 --- a/pkg/providers/tinkerbell/hardware/yaml.go +++ b/pkg/providers/tinkerbell/hardware/yaml.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "path/filepath" "sigs.k8s.io/yaml" ) @@ -84,6 +85,11 @@ func marshalSecretYAML(m Machine) ([]byte, error) { // os.Stdout is returned. func CreateOrStdout(path string) (*os.File, error) { if path != "" { + dir := filepath.Dir(path) + err := os.MkdirAll(dir, 0o755) + if err != nil { + return nil, fmt.Errorf("failed to create hardware yaml file: %v", err) + } return os.Create(path) } return os.Stdout, nil diff --git a/test/e2e/forceflow_test.go b/test/e2e/forceflow_test.go deleted file mode 100644 index 13824fb79624..000000000000 --- a/test/e2e/forceflow_test.go +++ /dev/null @@ -1,56 +0,0 @@ -//go:build e2e -// +build e2e - -package e2e - -import ( - "testing" - - "github.com/aws/eks-anywhere/internal/pkg/api" - "github.com/aws/eks-anywhere/pkg/api/v1alpha1" - "github.com/aws/eks-anywhere/test/framework" -) - -func runTinkerbellForceFlow(test *framework.ClusterE2ETest) { - test.GenerateClusterConfig() - test.GenerateHardwareConfig() - //test.PowerOnHardware() - //test.CreateCluster(framework.WithForce()) - //test.DeleteCluster() - //test.ValidateHardwareDecommissioned() -} - -func TestTinkerbellKubernetes121ForceFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), - framework.WithHardware(2), - ) - runTinkerbellForceFlow(test) -} - -func TestTinkerbellKubernetes122ForceFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewTinkerbell(t, framework.WithUbuntu122Tinkerbell()), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube122)), - framework.WithHardware(2), - ) - runTinkerbellForceFlow(test) -} - -func TestTinkerbellKubernetes121ThreeReplicasTwoWorkersForceFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), - framework.WithClusterFiller(api.WithWorkerNodeCount(2)), - framework.WithClusterFiller(api.WithControlPlaneCount(3)), - framework.WithHardware(5), - ) - runTinkerbellForceFlow(test) -} diff --git a/test/e2e/simpleflow_test.go b/test/e2e/simpleflow_test.go index d56ca1aa656d..dfb9158e8299 100644 --- a/test/e2e/simpleflow_test.go +++ b/test/e2e/simpleflow_test.go @@ -31,10 +31,10 @@ func runHelmInstallSimpleFlow(test *framework.ClusterE2ETest) { func runTinkerbellSimpleFlow(test *framework.ClusterE2ETest) { test.GenerateClusterConfig() test.GenerateHardwareConfig() - //test.PowerOffHardware() - //test.CreateCluster() - //test.DeleteCluster() - //test.ValidateHardwareDecommissioned() + test.PowerOnHardware() + test.CreateCluster(framework.WithForce()) + test.DeleteCluster() + test.ValidateHardwareDecommissioned() } func runSimpleFlow(test *framework.ClusterE2ETest) { @@ -244,61 +244,28 @@ func TestTinkerbellKubernetes121ExternalEtcdSimpleFlow(t *testing.T) { runTinkerbellSimpleFlow(test) } -func TestTinkerbellKubernetes121ExternalEtcdThreeReplicasTwoWorkersSimpleFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell(), framework.WithTinkerbellExternalEtcdTopology(1)), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), - framework.WithClusterFiller(api.WithWorkerNodeCount(2)), - framework.WithClusterFiller(api.WithControlPlaneCount(3)), - framework.WithHardware(6), - ) - runTinkerbellSimpleFlow(test) -} - -func TestTinkerbellKubernetes121ThreeReplicasTwoWorkersSimpleFlow(t *testing.T) { +func TestTinkerbellKubernetes121ThreeReplicasSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), - framework.WithClusterFiller(api.WithWorkerNodeCount(2)), + framework.WithClusterFiller(api.WithWorkerNodeCount(1)), framework.WithClusterFiller(api.WithControlPlaneCount(3)), - framework.WithHardware(5), - ) - runTinkerbellSimpleFlow(test) -} - -func TestTinkerbellKubernetes121SuperMicroSimpleFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), - framework.WithHardware(2), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), - ) - runTinkerbellSimpleFlow(test) -} - -func TestTinkerbellKubernetes121DellSimpleFlow(t *testing.T) { - test := framework.NewClusterE2ETest( - t, - framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), - framework.WithHardware(2), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), - framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), + framework.WithHardware(4), ) runTinkerbellSimpleFlow(test) } -func TestTinkerbellKubernetes121HPSimpleFlow(t *testing.T) { +func TestTinkerbellKubernetes121ThreeWorkersSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), - framework.WithHardware(2), framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), + framework.WithClusterFiller(api.WithWorkerNodeCount(3)), + framework.WithClusterFiller(api.WithControlPlaneCount(1)), + framework.WithHardware(4), ) runTinkerbellSimpleFlow(test) } diff --git a/test/framework/cluster.go b/test/framework/cluster.go index 3d6aea45aab7..592cfedf3b4b 100644 --- a/test/framework/cluster.go +++ b/test/framework/cluster.go @@ -16,6 +16,8 @@ import ( "testing" "time" + rapi "github.com/tinkerbell/rufio/api/v1alpha1" + rctrl "github.com/tinkerbell/rufio/controllers" "sigs.k8s.io/yaml" "github.com/aws/eks-anywhere/internal/pkg/api" @@ -226,15 +228,126 @@ func (e *ClusterE2ETest) GenerateClusterConfig(opts ...CommandOpt) { } func (e *ClusterE2ETest) PowerOffHardware() { - // TODO(chrisdoherty4) Requires an implementation that's independent of the old PBnJ service. + // Initializing BMC Client + ctx := context.Background() + bmcClientFactory := rctrl.NewBMCClientFactoryFunc(ctx) + + for _, h := range e.TestHardware { + bmcClient, err := bmcClientFactory(ctx, h.BMCIPAddress, "623", h.BMCUsername, h.BMCPassword) + if err != nil { + e.T.Fatalf("failed to create bmc client: %v", err) + } + + defer func() { + // Close BMC connection after reconcilation + err = bmcClient.Close(ctx) + if err != nil { + e.T.Fatalf("BMC close connection failed: %v", err) + } + }() + + _, err = bmcClient.SetPowerState(ctx, string(rapi.Off)) + if err != nil { + e.T.Fatalf("failed to power off hardware: %v", err) + } + } +} + +func (e *ClusterE2ETest) PXEBootHardware() { + // Initializing BMC Client + ctx := context.Background() + bmcClientFactory := rctrl.NewBMCClientFactoryFunc(ctx) + + for _, h := range e.TestHardware { + bmcClient, err := bmcClientFactory(ctx, h.BMCIPAddress, "623", h.BMCUsername, h.BMCPassword) + if err != nil { + e.T.Fatalf("failed to create bmc client: %v", err) + } + + defer func() { + // Close BMC connection after reconcilation + err = bmcClient.Close(ctx) + if err != nil { + e.T.Fatalf("BMC close connection failed: %v", err) + } + }() + + _, err = bmcClient.SetBootDevice(ctx, string(rapi.PXE), false, true) + if err != nil { + e.T.Fatalf("failed to set boot device to PXE on hardware: %v", err) + } + } } func (e *ClusterE2ETest) PowerOnHardware() { - // TODO(chrisdoherty4) Requires an implementation that's independent of the old PBnJ service. + // Initializing BMC Client + ctx := context.Background() + bmcClientFactory := rctrl.NewBMCClientFactoryFunc(ctx) + + for _, h := range e.TestHardware { + bmcClient, err := bmcClientFactory(ctx, h.BMCIPAddress, "623", h.BMCUsername, h.BMCPassword) + if err != nil { + e.T.Fatalf("failed to create bmc client: %v", err) + } + + defer func() { + // Close BMC connection after reconcilation + err = bmcClient.Close(ctx) + if err != nil { + e.T.Fatalf("BMC close connection failed: %v", err) + } + }() + + _, err = bmcClient.SetPowerState(ctx, string(rapi.On)) + if err != nil { + e.T.Fatalf("failed to power on hardware: %v", err) + } + } } func (e *ClusterE2ETest) ValidateHardwareDecommissioned() { - // TODO(chrisdoherty4) Requires an implementation that's independent of the old PBnJ service. + // Initializing BMC Client + ctx := context.Background() + bmcClientFactory := rctrl.NewBMCClientFactoryFunc(ctx) + + var failedToDecomm []*api.Hardware + for _, h := range e.TestHardware { + bmcClient, err := bmcClientFactory(ctx, h.BMCIPAddress, "443", h.BMCUsername, h.BMCPassword) + if err != nil { + e.T.Fatalf("failed to create bmc client: %v", err) + } + + defer func() { + // Close BMC connection after reconcilation + err = bmcClient.Close(ctx) + if err != nil { + e.T.Fatalf("BMC close connection failed: %v", err) + } + }() + + powerState, err := bmcClient.GetPowerState(ctx) + // add sleep retries to give the machine time to power off + timeout := 15 + for powerState != string(rapi.Off) && timeout > 0 { + if err != nil { + e.T.Logf("failed to get power state for hardware (%v): %v", h, err) + } + time.Sleep(5 * time.Second) + timeout = timeout - 5 + powerState, err = bmcClient.GetPowerState(ctx) + } + + if powerState != string(rapi.Off) { + e.T.Logf("failed to decommission hardware: id=%s, hostname=%s, bmc_ip=%s", h.MACAddress, h.Hostname, h.BMCIPAddress) + failedToDecomm = append(failedToDecomm, h) + } else { + e.T.Logf("successfully decommissioned hardware: id=%s, hostname=%s, bmc_ip=%s", h.MACAddress, h.Hostname, h.BMCIPAddress) + } + } + + if len(failedToDecomm) > 0 { + e.T.Fatalf("failed to decommision hardware during cluster deletion") + } } func (e *ClusterE2ETest) GenerateHardwareConfig(opts ...CommandOpt) { @@ -257,9 +370,8 @@ func (e *ClusterE2ETest) generateHardwareConfig(opts ...CommandOpt) { generateHardwareConfigArgs := []string{ "generate", "hardware", - "--skip-registration", - "-f", e.HardwareCsvLocation, - "-o", e.ClusterConfigFolder, + "-z", e.HardwareCsvLocation, + "-o", e.HardwareConfigLocation, } e.RunEKSA(generateHardwareConfigArgs, opts...) @@ -277,7 +389,7 @@ func (e *ClusterE2ETest) GenerateClusterConfigForVersion(eksaVersion string, opt e.buildClusterConfigFile() e.cleanup(func() { - os.Remove(e.ClusterConfigLocation) + //os.Remove(e.ClusterConfigLocation) }) } @@ -327,7 +439,7 @@ func (e *ClusterE2ETest) createCluster(opts ...CommandOpt) { } if e.Provider.Name() == TinkerbellProviderName { - createClusterArgs = append(createClusterArgs, "-w", e.HardwareConfigLocation) + createClusterArgs = append(createClusterArgs, "-z", e.HardwareCsvLocation) } e.RunEKSA(createClusterArgs, opts...) diff --git a/test/framework/tinkerbell.go b/test/framework/tinkerbell.go index 2a8e58eb3d47..1ecf69a82cdb 100644 --- a/test/framework/tinkerbell.go +++ b/test/framework/tinkerbell.go @@ -44,15 +44,22 @@ type Tinkerbell struct { func NewTinkerbell(t *testing.T, opts ...TinkerbellOpt) *Tinkerbell { checkRequiredEnvVars(t, requiredTinkerbellEnvVars) + cidr := os.Getenv(tinkerbellNetworkCidrEnvVar) + + tinkIP, err := GenerateUniqueIp(cidr) + if err != nil { + t.Fatalf("failed to generate tinkerbell ip from cidr %s: %v", cidr, err) + } + tink := &Tinkerbell{ t: t, fillers: []api.TinkerbellFiller{ - api.WithStringFromEnvVarTinkerbell(tinkerbellServerEnvVar, api.WithTinkerbellServer), + api.WithTinkerbellServer(tinkIP), api.WithStringFromEnvVarTinkerbell(tinkerbellSSHAuthorizedKey, api.WithSSHAuthorizedKeyForAllTinkerbellMachines), }, } - tink.cidr = os.Getenv(tinkerbellNetworkCidrEnvVar) + tink.cidr = cidr tink.inventoryCsvFilePath = os.Getenv(tinkerbellInventoryCsvFilePathEnvVar) for _, opt := range opts { @@ -87,7 +94,7 @@ func (t *Tinkerbell) customizeProviderConfig(file string, fillers ...api.Tinkerb func (t *Tinkerbell) ClusterConfigFillers() []api.ClusterFiller { clusterIP, err := GenerateUniqueIp(t.cidr) if err != nil { - t.t.Fatalf("failed to generate ip for tinkerbell cidr %s: %v", t.cidr, err) + t.t.Fatalf("failed to generate cluster ip from cidr %s: %v", t.cidr, err) } t.clusterFillers = append(t.clusterFillers, api.WithControlPlaneEndpointIP(clusterIP)) From 97f47a373a965facb100531350c265c61152f3d7 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Tue, 7 Jun 2022 12:19:28 -0400 Subject: [PATCH 05/27] provision tink boostrap and server ip for ci --- internal/test/e2e/tinkerbell.go | 9 +++++++-- test/framework/tinkerbell.go | 15 +++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/internal/test/e2e/tinkerbell.go b/internal/test/e2e/tinkerbell.go index 7c8836d83cca..0335f100d07e 100644 --- a/internal/test/e2e/tinkerbell.go +++ b/internal/test/e2e/tinkerbell.go @@ -17,6 +17,7 @@ const ( tinkerbellTestsRe = `^.*Tinkerbell.*$` e2eHardwareCsvFilePath = "e2e-inventory.csv" MaxHardwarePerE2ETestEnvVar = "T_TINKERBELL_MAX_HARDWARE_PER_TEST" + tinkerbellBootstrapInterfaceEnvVar = "T_TINKERBELL_BOOTSTRAP_INTERFACE" ) func (e *E2ESession) setupTinkerbellEnv(testRegex string) error { @@ -58,7 +59,11 @@ func (e *E2ESession) setupTinkerbellEnv(testRegex string) error { return fmt.Errorf("failed to download tinkerbell inventory file (%s) to test instance : %v", inventoryFileName, err) } - tinkInterface := "ens192" + tinkInterface := os.Getenv(tinkerbellBootstrapInterfaceEnvVar) + if tinkInterface == "" { + return fmt.Errorf("tinkerbell bootstrap interface env var is required: %s", tinkerbellBootstrapInterfaceEnvVar) + } + err = e.setTinkerbellBootstrapIPInInstance(tinkInterface) if err != nil { return fmt.Errorf("failed to set tinkerbell boostrap ip on interface (%s) in test instance : %v", tinkInterface, err) @@ -72,7 +77,7 @@ func (e *E2ESession) setupTinkerbellEnv(testRegex string) error { func (e *E2ESession) setTinkerbellBootstrapIPInInstance(tinkInterface string) error { logger.V(1).Info("Setting Tinkerbell Bootstrap IP in instance") - command := fmt.Sprintf("export TINKERBELL_BOOSTRAP_IP=$(/sbin/ip -o -4 addr list %s | awk '{print $4}' | cut -d/ -f1) && echo TINKERBELL_BOOTSTRAP_IP=$TINKERBELL_BOOSTRAP_IP | tee -a /etc/environment", tinkInterface) + command := fmt.Sprintf("export T_TINKERBELL_BOOTSTRAP_IP=$(/sbin/ip -o -4 addr list %s | awk '{print $4}' | cut -d/ -f1) && echo T_TINKERBELL_BOOTSTRAP_IP=$T_TINKERBELL_BOOTSTRAP_IP | tee -a /etc/environment", tinkInterface) if err := ssm.Run(e.session, e.instanceId, command); err != nil { return fmt.Errorf("setting tinkerbell boostrap ip: %v", err) } diff --git a/test/framework/tinkerbell.go b/test/framework/tinkerbell.go index 1ecf69a82cdb..49c4ab98caf3 100644 --- a/test/framework/tinkerbell.go +++ b/test/framework/tinkerbell.go @@ -10,7 +10,7 @@ import ( const ( TinkerbellProviderName = "tinkerbell" - tinkerbellServerEnvVar = "T_TINKERBELL_IP" + tinkerbellBootstrapIPEnvVar = "T_TINKERBELL_BOOTSTRAP_IP" tinkerbellNetworkCidrEnvVar = "T_TINKERBELL_NETWORK_CIDR" tinkerbellImageUbuntu120EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_20" tinkerbellImageUbuntu121EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_21" @@ -20,7 +20,6 @@ const ( ) var requiredTinkerbellEnvVars = []string{ - tinkerbellServerEnvVar, tinkerbellNetworkCidrEnvVar, tinkerbellImageUbuntu120EnvVar, tinkerbellImageUbuntu121EnvVar, @@ -38,6 +37,8 @@ type Tinkerbell struct { t *testing.T fillers []api.TinkerbellFiller clusterFillers []api.ClusterFiller + serverIP string + bootstrapIP string cidr string inventoryCsvFilePath string } @@ -46,7 +47,7 @@ func NewTinkerbell(t *testing.T, opts ...TinkerbellOpt) *Tinkerbell { checkRequiredEnvVars(t, requiredTinkerbellEnvVars) cidr := os.Getenv(tinkerbellNetworkCidrEnvVar) - tinkIP, err := GenerateUniqueIp(cidr) + serverIP, err := GenerateUniqueIp(cidr) if err != nil { t.Fatalf("failed to generate tinkerbell ip from cidr %s: %v", cidr, err) } @@ -54,11 +55,17 @@ func NewTinkerbell(t *testing.T, opts ...TinkerbellOpt) *Tinkerbell { tink := &Tinkerbell{ t: t, fillers: []api.TinkerbellFiller{ - api.WithTinkerbellServer(tinkIP), + api.WithTinkerbellServer(serverIP), api.WithStringFromEnvVarTinkerbell(tinkerbellSSHAuthorizedKey, api.WithSSHAuthorizedKeyForAllTinkerbellMachines), }, } + tink.serverIP = serverIP + tink.bootstrapIP = os.Getenv(tinkerbellBootstrapIPEnvVar) + if tink.bootstrapIP == "" { + t.Fatalf("tinkerbell bootstrap ip is required!") + } + tink.cidr = cidr tink.inventoryCsvFilePath = os.Getenv(tinkerbellInventoryCsvFilePathEnvVar) From 1346b66e0d89f9019e104bd9318ecd2e551f1226 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Tue, 7 Jun 2022 13:40:05 -0400 Subject: [PATCH 06/27] update codebuild job for tinkerbell ci --- .../build/buildspecs/test-eks-a-cli.yml | 28 ++++++++++++-- .../build/script/create_infra_config.sh | 37 +++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) create mode 100755 cmd/integration_test/build/script/create_infra_config.sh diff --git a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml index e8b2bcd617ed..8244fc669540 100644 --- a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml @@ -5,10 +5,14 @@ env: INTEGRATION_TEST_MAX_EC2_COUNT: 130 INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 130 T_CLOUDSTACK_CIDR: "10.11.255.0/24" - SKIPPED_TESTS: "TestVSphereKubernetes122BottlerocketAutoimport,TestTinkerbellKubernetes121ForceFlow,TestTinkerbellKubernetes121ThreeReplicasTwoWorkersForceFlow,TestTinkerbellKubernetes122ForceFlow,TestTinkerbellKubernetes120SimpleFlow,TestTinkerbellKubernetes121SimpleFlow,TestTinkerbellKubernetes121ThreeReplicasTwoWorkersSimpleFlow,TestTinkerbellKubernetes121DellSimpleFlow,TestTinkerbellKubernetes121HPSimpleFlow,TestTinkerbellKubernetes121SuperMicroSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdThreeReplicasTwoWorkersSimpleFlow,TestCloudStackUpgradeMulticlusterWorkloadClusterWithFluxLegacy,TestCloudStackKubernetes121StackedEtcdRedhat,TestCloudStackKubernetes120RedhatWorkerNodeUpgrade,TestCloudStackKubernetes120RedhatTo121MultipleFieldsUpgrade,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC,TestSnowKubernetes121UbuntuProxyConfig" + SKIPPED_TESTS: "TestVSphereKubernetes122BottlerocketAutoimport,TestCloudStackUpgradeMulticlusterWorkloadClusterWithFluxLegacy,TestCloudStackKubernetes121StackedEtcdRedhat,TestCloudStackKubernetes120RedhatWorkerNodeUpgrade,TestCloudStackKubernetes120RedhatTo121MultipleFieldsUpgrade,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC,TestSnowKubernetes121UbuntuProxyConfig" CLOUDSTACK_PROVIDER: true EKSA_GIT_KNOWN_HOSTS: "/tmp/known_hosts" EKSA_GIT_PRIVATE_KEY: "/tmp/private-key" + T_TINKERBELL_MAX_HARDWARE_PER_TEST: 4 + T_TINKERBELL_INVENTORY_CSV: "hardware-manifests/inventory.csv" + T_TINKERBELL_BOOTSTRAP_INTERFACE: "ens192" + INTEGRATION_TEST_INFRA_CONFIG: "/tmp/test-infra.yml" secrets-manager: EKSA_VSPHERE_USERNAME: "vsphere_ci_beta_connection:vsphere_username" EKSA_VSPHERE_PASSWORD: "vsphere_ci_beta_connection:vsphere_password" @@ -65,11 +69,30 @@ env: T_CLOUDSTACK_POD_CIDR: "cloudstack_ci_beta_connection:pod_cidr" T_CLOUDSTACK_SERVICE_CIDR: "cloudstack_ci_beta_connection:service_cidr" T_CLOUDSTACK_SSH_AUTHORIZED_KEY: "vsphere_ci_beta_connection:ssh_authorized_key" + T_TINKERBELL_IMAGE_UBUNTU_1_20: "tinkerbell_ci:image_ubuntu_1_20" + T_TINKERBELL_IMAGE_UBUNTU_1_21: "tinkerbell_ci:image_ubuntu_1_21" + T_TINKERBELL_IMAGE_UBUNTU_1_22: "tinkerbell_ci:image_ubuntu_1_22" + T_TINKERBELL_IMAGE_UBUNTU_1_23: "tinkerbell_ci:image_ubuntu_1_23" + T_TINKERBELL_SSH_AUTHORIZED_KEY: "tinkerbell_ci:ssh_key" + T_TINKERBELL_NETWORK_CIDR: "tinkerbell_ci:network_cidr" + T_TINKERBELL_S3_INVENTORY_CSV_KEY: "tinkerbell_ci:s3_inventory_csv" + TEST_RUNNER_GOVC_USERNAME: "tinkerbell_ci:govc_username" + TEST_RUNNER_GOVC_PASSWORD: "tinkerbell_ci: govc_password" + TEST_RUNNER_GOVC_URL: "tinkerbell_ci: govc_url" + TEST_RUNNER_GOVC_LIBRARY: "tinkerbell_ci: govc_library" + TEST_RUNNER_GOVC_TEMPLATE: "tinkerbell_ci: govc_template" + TEST_RUNNER_GOVC_DATACENTER: "tinkerbell_ci: govc_datacenter" + TEST_RUNNER_GOVC_DATASTORE: "tinkerbell_ci: govc_datastore" + TEST_RUNNER_GOVC_RESOURCE_POOL: "tinkerbell_ci: govc_resource_pool" + TEST_RUNNER_GOVC_NETWORK: "tinkerbell_ci: govc_network" + TEST_RUNNER_GOVC_FOLDER: "tinkerbell_ci: govc_folder" + phases: pre_build: commands: - source ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/setup_profile.sh + - source ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/create_infra_config.sh - ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/start_docker.sh - export CLUSTER_NAME_PREFIX="${BRANCH_NAME//./-}" - > @@ -86,11 +109,10 @@ phases: fi - > ./bin/test e2e run - -a ${INTEGRATION_TEST_AL2_AMI_ID} + -c ${INTEGRATION_TEST_INFRA_CONFIG} -s ${INTEGRATION_TEST_STORAGE_BUCKET} -j ${JOB_ID} -i ${INTEGRATION_TEST_INSTANCE_PROFILE} - -n ${INTEGRATION_TEST_SUBNET_ID} -m ${INTEGRATION_TEST_MAX_EC2_COUNT} -p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT} -r 'Test' diff --git a/cmd/integration_test/build/script/create_infra_config.sh b/cmd/integration_test/build/script/create_infra_config.sh new file mode 100755 index 000000000000..438735076e30 --- /dev/null +++ b/cmd/integration_test/build/script/create_infra_config.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -x +set -o pipefail + +cat << EOF > ${INTEGRATION_TEST_INFRA_CONFIG} +--- + +ec2: + amiId: ${INTEGRATION_TEST_AL2_AMI_ID} + subnetId: ${INTEGRATION_TEST_SUBNET_ID} + +vSphere: + url: ${TEST_RUNNER_GOVC_URL} + insecure: True + library: ${TEST_RUNNER_GOVC_LIBRARY} + template: ${TEST_RUNNER_GOVC_TEMPLATE} + datacenter: ${TEST_RUNNER_GOVC_DATACENTER} + datastore: ${TEST_RUNNER_GOVC_DATASTORE} + resourcePool: ${TEST_RUNNER_GOVC_RESOURCE_POOL} + network: ${TEST_RUNNER_GOVC_NETWORK} + folder: ${TEST_RUNNER_GOVC_FOLDER} +EOF From 6fb363e74a989d53b1e6e19a63b15feb25eedcb2 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Tue, 7 Jun 2022 13:47:35 -0400 Subject: [PATCH 07/27] fix linter --- internal/test/e2e/run.go | 1 + internal/test/e2e/testRunner.go | 3 ++- test/framework/cluster.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/test/e2e/run.go b/internal/test/e2e/run.go index 42b441ab5af5..9f8b0ecdac33 100644 --- a/internal/test/e2e/run.go +++ b/internal/test/e2e/run.go @@ -9,6 +9,7 @@ import ( "sync" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/eks-anywhere/internal/pkg/api" "github.com/aws/eks-anywhere/internal/pkg/s3" "github.com/aws/eks-anywhere/internal/pkg/ssm" diff --git a/internal/test/e2e/testRunner.go b/internal/test/e2e/testRunner.go index 6b1c406ce075..4a40c9464f5d 100644 --- a/internal/test/e2e/testRunner.go +++ b/internal/test/e2e/testRunner.go @@ -8,12 +8,13 @@ import ( "time" aws_ssm "github.com/aws/aws-sdk-go/service/ssm" + "gopkg.in/yaml.v2" + "github.com/aws/eks-anywhere/internal/pkg/ec2" "github.com/aws/eks-anywhere/internal/pkg/ssm" "github.com/aws/eks-anywhere/internal/pkg/vsphere" "github.com/aws/eks-anywhere/pkg/logger" "github.com/aws/eks-anywhere/pkg/retrier" - "gopkg.in/yaml.v2" ) const ( diff --git a/test/framework/cluster.go b/test/framework/cluster.go index 592cfedf3b4b..48e877496bf9 100644 --- a/test/framework/cluster.go +++ b/test/framework/cluster.go @@ -389,7 +389,7 @@ func (e *ClusterE2ETest) GenerateClusterConfigForVersion(eksaVersion string, opt e.buildClusterConfigFile() e.cleanup(func() { - //os.Remove(e.ClusterConfigLocation) + os.Remove(e.ClusterConfigLocation) }) } From 062e9106c30271f1ea632e57b8f0035a295c1b2d Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Tue, 7 Jun 2022 22:35:11 -0400 Subject: [PATCH 08/27] add kube 1.23 test --- test/e2e/simpleflow_test.go | 22 ++++++++++++++++++++++ test/framework/tinkerbell.go | 12 ++++++++++++ 2 files changed, 34 insertions(+) diff --git a/test/e2e/simpleflow_test.go b/test/e2e/simpleflow_test.go index dfb9158e8299..b720a9c27fd0 100644 --- a/test/e2e/simpleflow_test.go +++ b/test/e2e/simpleflow_test.go @@ -233,6 +233,28 @@ func TestTinkerbellKubernetes121SimpleFlow(t *testing.T) { runTinkerbellSimpleFlow(test) } +func TestTinkerbellKubernetes122SimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithUbuntu122Tinkerbell()), + framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube122)), + framework.WithHardware(2), + ) + runTinkerbellSimpleFlow(test) +} + +func TestTinkerbellKubernetes123SimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithUbuntu123Tinkerbell()), + framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube123)), + framework.WithHardware(2), + ) + runTinkerbellSimpleFlow(test) +} + func TestTinkerbellKubernetes121ExternalEtcdSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, diff --git a/test/framework/tinkerbell.go b/test/framework/tinkerbell.go index 49c4ab98caf3..da9424ed8be5 100644 --- a/test/framework/tinkerbell.go +++ b/test/framework/tinkerbell.go @@ -15,6 +15,7 @@ const ( tinkerbellImageUbuntu120EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_20" tinkerbellImageUbuntu121EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_21" tinkerbellImageUbuntu122EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_22" + tinkerbellImageUbuntu123EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_23" tinkerbellInventoryCsvFilePathEnvVar = "T_TINKERBELL_INVENTORY_CSV" tinkerbellSSHAuthorizedKey = "T_TINKERBELL_SSH_AUTHORIZED_KEY" ) @@ -23,6 +24,8 @@ var requiredTinkerbellEnvVars = []string{ tinkerbellNetworkCidrEnvVar, tinkerbellImageUbuntu120EnvVar, tinkerbellImageUbuntu121EnvVar, + tinkerbellImageUbuntu122EnvVar, + tinkerbellImageUbuntu123EnvVar, tinkerbellInventoryCsvFilePathEnvVar, tinkerbellSSHAuthorizedKey, } @@ -135,6 +138,15 @@ func WithUbuntu122Tinkerbell() TinkerbellOpt { } } +func WithUbuntu123Tinkerbell() TinkerbellOpt { + return func(t *Tinkerbell) { + t.fillers = append(t.fillers, + api.WithStringFromEnvVarTinkerbell(tinkerbellImageUbuntu123EnvVar, api.WithImageUrlForAllTinkerbellMachines), + api.WithOsFamilyForAllTinkerbellMachines(anywherev1.Ubuntu), + ) + } +} + func WithTinkerbellExternalEtcdTopology(count int) TinkerbellOpt { return func(t *Tinkerbell) { t.fillers = append([]api.TinkerbellFiller{api.WithTinkerbellEtcdMachineConfig()}, t.fillers...) From 24785df515efb71156d748dd6a2fcf3d51f058cf Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Tue, 7 Jun 2022 22:51:36 -0400 Subject: [PATCH 09/27] account for future tinkerbell boostrap ip arg during create cluster cmd --- test/framework/cluster.go | 7 +++++++ test/framework/tinkerbell.go | 5 ----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test/framework/cluster.go b/test/framework/cluster.go index 48e877496bf9..920e5d264d44 100644 --- a/test/framework/cluster.go +++ b/test/framework/cluster.go @@ -440,6 +440,13 @@ func (e *ClusterE2ETest) createCluster(opts ...CommandOpt) { if e.Provider.Name() == TinkerbellProviderName { createClusterArgs = append(createClusterArgs, "-z", e.HardwareCsvLocation) + + tinkBootstrapIP := os.Getenv(tinkerbellBootstrapIPEnvVar) + if tinkBootstrapIP == "" { + e.T.Fatalf("tinkerbell bootstrap ip is required!") + } + // TODO: uncomment once boostrap ip arg is introduced into the CLI + //createClusterArgs = append(createClusterArgs, "-tbd") } e.RunEKSA(createClusterArgs, opts...) diff --git a/test/framework/tinkerbell.go b/test/framework/tinkerbell.go index da9424ed8be5..714a33276040 100644 --- a/test/framework/tinkerbell.go +++ b/test/framework/tinkerbell.go @@ -41,7 +41,6 @@ type Tinkerbell struct { fillers []api.TinkerbellFiller clusterFillers []api.ClusterFiller serverIP string - bootstrapIP string cidr string inventoryCsvFilePath string } @@ -64,10 +63,6 @@ func NewTinkerbell(t *testing.T, opts ...TinkerbellOpt) *Tinkerbell { } tink.serverIP = serverIP - tink.bootstrapIP = os.Getenv(tinkerbellBootstrapIPEnvVar) - if tink.bootstrapIP == "" { - t.Fatalf("tinkerbell bootstrap ip is required!") - } tink.cidr = cidr tink.inventoryCsvFilePath = os.Getenv(tinkerbellInventoryCsvFilePathEnvVar) From b60040bf25b4274526ba475c8ea7118f7eba8f8b Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Wed, 8 Jun 2022 10:35:52 -0400 Subject: [PATCH 10/27] remove tink feature flag enable --- test/e2e/simpleflow_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/e2e/simpleflow_test.go b/test/e2e/simpleflow_test.go index b720a9c27fd0..10d56064374f 100644 --- a/test/e2e/simpleflow_test.go +++ b/test/e2e/simpleflow_test.go @@ -215,7 +215,6 @@ func TestTinkerbellKubernetes120SimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, framework.NewTinkerbell(t, framework.WithUbuntu120Tinkerbell()), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube120)), framework.WithHardware(2), ) @@ -226,7 +225,6 @@ func TestTinkerbellKubernetes121SimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), framework.WithHardware(2), ) @@ -237,7 +235,6 @@ func TestTinkerbellKubernetes122SimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, framework.NewTinkerbell(t, framework.WithUbuntu122Tinkerbell()), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube122)), framework.WithHardware(2), ) @@ -248,7 +245,6 @@ func TestTinkerbellKubernetes123SimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, framework.NewTinkerbell(t, framework.WithUbuntu123Tinkerbell()), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube123)), framework.WithHardware(2), ) @@ -259,7 +255,6 @@ func TestTinkerbellKubernetes121ExternalEtcdSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell(), framework.WithTinkerbellExternalEtcdTopology(1)), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), framework.WithHardware(3), ) @@ -270,7 +265,6 @@ func TestTinkerbellKubernetes121ThreeReplicasSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), framework.WithClusterFiller(api.WithWorkerNodeCount(1)), framework.WithClusterFiller(api.WithControlPlaneCount(3)), @@ -283,7 +277,6 @@ func TestTinkerbellKubernetes121ThreeWorkersSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), - framework.WithEnvVar("TINKERBELL_PROVIDER", "true"), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), framework.WithClusterFiller(api.WithWorkerNodeCount(3)), framework.WithClusterFiller(api.WithControlPlaneCount(1)), From 138fd8a506b43740b230a6f1a1d2b4803fa29d38 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 9 Jun 2022 09:12:44 -0400 Subject: [PATCH 11/27] add option to override the local tinkerbell IP in the bootstrap cluster --- internal/test/cleanup/cleanup.go | 4 ++-- internal/test/e2e/testRunner.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/test/cleanup/cleanup.go b/internal/test/cleanup/cleanup.go index c346164fed50..c72a91bfe287 100644 --- a/internal/test/cleanup/cleanup.go +++ b/internal/test/cleanup/cleanup.go @@ -63,7 +63,7 @@ func CleanUpVsphereTestResources(ctx context.Context, clusterName string) error if err != nil { return fmt.Errorf("validating cluster name: %v", err) } - err = vsphereRmVms(ctx, clusterName) + err = VsphereRmVms(ctx, clusterName) if err != nil { return fmt.Errorf("removing vcenter vms: %v", err) } @@ -71,7 +71,7 @@ func CleanUpVsphereTestResources(ctx context.Context, clusterName string) error return nil } -func vsphereRmVms(ctx context.Context, clusterName string) error { +func VsphereRmVms(ctx context.Context, clusterName string) error { logger.V(1).Info("Deleting vsphere vcenter vms") executableBuilder, close, err := executables.NewExecutableBuilder(ctx, executables.DefaultEksaImage()) if err != nil { diff --git a/internal/test/e2e/testRunner.go b/internal/test/e2e/testRunner.go index 4a40c9464f5d..880412973c0b 100644 --- a/internal/test/e2e/testRunner.go +++ b/internal/test/e2e/testRunner.go @@ -13,6 +13,7 @@ import ( "github.com/aws/eks-anywhere/internal/pkg/ec2" "github.com/aws/eks-anywhere/internal/pkg/ssm" "github.com/aws/eks-anywhere/internal/pkg/vsphere" + "github.com/aws/eks-anywhere/internal/test/cleanup" "github.com/aws/eks-anywhere/pkg/logger" "github.com/aws/eks-anywhere/pkg/retrier" ) @@ -209,7 +210,7 @@ func (e *Ec2TestRunner) tagInstance(c instanceRunConf, key, value string) error func (v *VSphereTestRunner) decommInstance(c instanceRunConf) error { _, deregisterError := ssm.DeregisterInstance(c.session, v.InstanceID) _, deactivateError := ssm.DeleteActivation(c.session, v.ActivationId) - deleteError := vsphereRmVms(context.Background(), getTestRunnerName(c.jobId)) + deleteError := cleanup.VsphereRmVms(context.Background(), getTestRunnerName(c.jobId)) if deregisterError != nil { return fmt.Errorf("failed to decommission vsphere test runner ssm instance: %v", deregisterError) From 11c3c4604a8a72c8cc0b0fcd7dc65050e48c32ba Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 9 Jun 2022 09:47:25 -0400 Subject: [PATCH 12/27] update tests for bootstrap flag --- test/e2e/simpleflow_test.go | 2 +- test/framework/cluster.go | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/test/e2e/simpleflow_test.go b/test/e2e/simpleflow_test.go index 10d56064374f..657b46785f5d 100644 --- a/test/e2e/simpleflow_test.go +++ b/test/e2e/simpleflow_test.go @@ -31,7 +31,7 @@ func runHelmInstallSimpleFlow(test *framework.ClusterE2ETest) { func runTinkerbellSimpleFlow(test *framework.ClusterE2ETest) { test.GenerateClusterConfig() test.GenerateHardwareConfig() - test.PowerOnHardware() + //test.PowerOnHardware() test.CreateCluster(framework.WithForce()) test.DeleteCluster() test.ValidateHardwareDecommissioned() diff --git a/test/framework/cluster.go b/test/framework/cluster.go index 920e5d264d44..7c6af8013501 100644 --- a/test/framework/cluster.go +++ b/test/framework/cluster.go @@ -440,13 +440,10 @@ func (e *ClusterE2ETest) createCluster(opts ...CommandOpt) { if e.Provider.Name() == TinkerbellProviderName { createClusterArgs = append(createClusterArgs, "-z", e.HardwareCsvLocation) - tinkBootstrapIP := os.Getenv(tinkerbellBootstrapIPEnvVar) - if tinkBootstrapIP == "" { - e.T.Fatalf("tinkerbell bootstrap ip is required!") + if tinkBootstrapIP != "" { + createClusterArgs = append(createClusterArgs, "--tinkerbell-bootstrap-ip", tinkBootstrapIP) } - // TODO: uncomment once boostrap ip arg is introduced into the CLI - //createClusterArgs = append(createClusterArgs, "-tbd") } e.RunEKSA(createClusterArgs, opts...) From 3c90c21a7f692ffe43826ce0a0a6211e243e57b0 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Fri, 10 Jun 2022 13:22:55 -0400 Subject: [PATCH 13/27] update to lab vcenter --- .../build/buildspecs/test-infra.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/integration_test/build/buildspecs/test-infra.yml b/cmd/integration_test/build/buildspecs/test-infra.yml index 5be622230e9d..9730c275022f 100644 --- a/cmd/integration_test/build/buildspecs/test-infra.yml +++ b/cmd/integration_test/build/buildspecs/test-infra.yml @@ -5,12 +5,12 @@ ec2: subnetId: subnet-0fd4b6ba7c9e17b5f vSphere: - url: https://vcenter.sddc-44-239-186-141.vmwarevmc.com + url: https://10.80.148.25 insecure: True library: eks-a-templates - template: eks-a-admin-ipmitool - datacenter: SDDC-Datacenter - datastore: WorkloadDatastore - resourcePool: Compute-ResourcePool - network: sddc-cgw-network-4 - folder: gwest \ No newline at end of file + template: eks-a-admin-ci + datacenter: Denver + datastore: datastore1.2 + resourcePool: CI + network: CI + folder: CI \ No newline at end of file From 6c0ba700d6b74561c518e2438cac2dda598ac5bb Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Fri, 10 Jun 2022 13:44:16 -0400 Subject: [PATCH 14/27] update max cluster name length and power off hardware prior to each test --- internal/pkg/api/hardware.go | 4 +++ internal/pkg/api/tinkerbell.go | 19 +++++++++++++ internal/test/e2e/setup.go | 15 ++++++++-- internal/test/e2e/tinkerbell.go | 2 +- test/e2e/simpleflow_test.go | 24 ++++++++++------ test/framework/cluster.go | 49 +++++++++++++-------------------- test/framework/tinkerbell.go | 1 + 7 files changed, 72 insertions(+), 42 deletions(-) diff --git a/internal/pkg/api/hardware.go b/internal/pkg/api/hardware.go index e86046ee99f4..40e7a7e20693 100644 --- a/internal/pkg/api/hardware.go +++ b/internal/pkg/api/hardware.go @@ -16,6 +16,10 @@ const ( HardwareVendorHP = "hp" HardwareVendorSuperMicro = "supermicro" HardwareVendorUnspecified = "unspecified" + HardwareLabelTypeKeyName = "type" + ControlPlane = "control-plane" + Worker = "worker" + ExternalEtcd = "etcd" ) // Alias for backwards compatibility. diff --git a/internal/pkg/api/tinkerbell.go b/internal/pkg/api/tinkerbell.go index 8ca13d12e8d6..cc4b4457d304 100644 --- a/internal/pkg/api/tinkerbell.go +++ b/internal/pkg/api/tinkerbell.go @@ -129,6 +129,24 @@ func WithSSHAuthorizedKeyForAllTinkerbellMachines(key string) TinkerbellFiller { } } +func WithHardwareSelectorLabels() TinkerbellFiller { + return func(config TinkerbellConfig) error { + clusterName := config.clusterConfig.Name + cpName := providers.GetControlPlaneNodeName(clusterName) + workerName := clusterName + + cpMachineConfig := config.machineConfigs[cpName] + cpMachineConfig.Spec.HardwareSelector = map[string]string{HardwareLabelTypeKeyName: ControlPlane} + config.machineConfigs[cpName] = cpMachineConfig + + workerMachineConfig := config.machineConfigs[workerName] + workerMachineConfig.Spec.HardwareSelector = map[string]string{HardwareLabelTypeKeyName: Worker} + config.machineConfigs[workerName] = workerMachineConfig + + return nil + } +} + func WithTinkerbellEtcdMachineConfig() TinkerbellFiller { return func(config TinkerbellConfig) error { clusterName := config.clusterConfig.Name @@ -145,6 +163,7 @@ func WithTinkerbellEtcdMachineConfig() TinkerbellFiller { Name: name, }, Spec: anywherev1.TinkerbellMachineConfigSpec{ + HardwareSelector: map[string]string{HardwareLabelTypeKeyName: ExternalEtcd}, TemplateRef: anywherev1.Ref{ Name: clusterName, Kind: anywherev1.TinkerbellTemplateConfigKind, diff --git a/internal/test/e2e/setup.go b/internal/test/e2e/setup.go index 5752e21a90bd..425a66da5f35 100644 --- a/internal/test/e2e/setup.go +++ b/internal/test/e2e/setup.go @@ -236,15 +236,24 @@ func clusterPrefix(branch, instanceId string) (clusterPrefix string) { for _, char := range forbiddenChars { sanitizedBranch = strings.ReplaceAll(sanitizedBranch, char, "-") } + + if len(sanitizedBranch) > 7 { + sanitizedBranch = sanitizedBranch[:7] + } + + if len(instanceId) > 7 { + instanceId = instanceId[:7] + } + clusterPrefix = fmt.Sprintf("%s-%s", sanitizedBranch, instanceId) return clusterPrefix } func clusterName(branch, instanceId, testName string) (clusterName string) { clusterName = fmt.Sprintf("%s-%s", clusterPrefix(branch, instanceId), e2etests.GetTestNameHash(testName)) - if len(clusterName) > 80 { - logger.Info("Cluster name is longer than 80 characters; truncating to 80 characters.", "original cluster name", clusterName, "truncated cluster name", clusterName[:80]) - clusterName = clusterName[:80] + if len(clusterName) > 63 { + logger.Info("Cluster name is longer than 63 characters; truncating to 63 characters.", "original cluster name", clusterName, "truncated cluster name", clusterName[:63]) + clusterName = clusterName[:63] } return clusterName } diff --git a/internal/test/e2e/tinkerbell.go b/internal/test/e2e/tinkerbell.go index 0335f100d07e..07767ed0503e 100644 --- a/internal/test/e2e/tinkerbell.go +++ b/internal/test/e2e/tinkerbell.go @@ -77,7 +77,7 @@ func (e *E2ESession) setupTinkerbellEnv(testRegex string) error { func (e *E2ESession) setTinkerbellBootstrapIPInInstance(tinkInterface string) error { logger.V(1).Info("Setting Tinkerbell Bootstrap IP in instance") - command := fmt.Sprintf("export T_TINKERBELL_BOOTSTRAP_IP=$(/sbin/ip -o -4 addr list %s | awk '{print $4}' | cut -d/ -f1) && echo T_TINKERBELL_BOOTSTRAP_IP=$T_TINKERBELL_BOOTSTRAP_IP | tee -a /etc/environment", tinkInterface) + command := fmt.Sprintf("export T_TINKERBELL_BOOTSTRAP_IP=$(/sbin/ip -o -4 addr list %s | awk '{print $4}' | cut -d/ -f1) && echo T_TINKERBELL_BOOTSTRAP_IP=\"$T_TINKERBELL_BOOTSTRAP_IP\" | tee -a /etc/environment", tinkInterface) if err := ssm.Run(e.session, e.instanceId, command); err != nil { return fmt.Errorf("setting tinkerbell boostrap ip: %v", err) } diff --git a/test/e2e/simpleflow_test.go b/test/e2e/simpleflow_test.go index 657b46785f5d..9e1bb548593d 100644 --- a/test/e2e/simpleflow_test.go +++ b/test/e2e/simpleflow_test.go @@ -31,7 +31,7 @@ func runHelmInstallSimpleFlow(test *framework.ClusterE2ETest) { func runTinkerbellSimpleFlow(test *framework.ClusterE2ETest) { test.GenerateClusterConfig() test.GenerateHardwareConfig() - //test.PowerOnHardware() + test.PowerOffHardware() test.CreateCluster(framework.WithForce()) test.DeleteCluster() test.ValidateHardwareDecommissioned() @@ -216,7 +216,8 @@ func TestTinkerbellKubernetes120SimpleFlow(t *testing.T) { t, framework.NewTinkerbell(t, framework.WithUbuntu120Tinkerbell()), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube120)), - framework.WithHardware(2), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), ) runTinkerbellSimpleFlow(test) } @@ -226,7 +227,8 @@ func TestTinkerbellKubernetes121SimpleFlow(t *testing.T) { t, framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), - framework.WithHardware(2), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), ) runTinkerbellSimpleFlow(test) } @@ -236,7 +238,8 @@ func TestTinkerbellKubernetes122SimpleFlow(t *testing.T) { t, framework.NewTinkerbell(t, framework.WithUbuntu122Tinkerbell()), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube122)), - framework.WithHardware(2), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), ) runTinkerbellSimpleFlow(test) } @@ -246,7 +249,8 @@ func TestTinkerbellKubernetes123SimpleFlow(t *testing.T) { t, framework.NewTinkerbell(t, framework.WithUbuntu123Tinkerbell()), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube123)), - framework.WithHardware(2), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), ) runTinkerbellSimpleFlow(test) } @@ -256,7 +260,9 @@ func TestTinkerbellKubernetes121ExternalEtcdSimpleFlow(t *testing.T) { t, framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell(), framework.WithTinkerbellExternalEtcdTopology(1)), framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), - framework.WithHardware(3), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), + framework.WithExternalEtcdHardware(1), ) runTinkerbellSimpleFlow(test) } @@ -268,7 +274,8 @@ func TestTinkerbellKubernetes121ThreeReplicasSimpleFlow(t *testing.T) { framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), framework.WithClusterFiller(api.WithWorkerNodeCount(1)), framework.WithClusterFiller(api.WithControlPlaneCount(3)), - framework.WithHardware(4), + framework.WithControlPlaneHardware(3), + framework.WithWorkerHardware(1), ) runTinkerbellSimpleFlow(test) } @@ -280,7 +287,8 @@ func TestTinkerbellKubernetes121ThreeWorkersSimpleFlow(t *testing.T) { framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), framework.WithClusterFiller(api.WithWorkerNodeCount(3)), framework.WithClusterFiller(api.WithControlPlaneCount(1)), - framework.WithHardware(4), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(3), ) runTinkerbellSimpleFlow(test) } diff --git a/test/framework/cluster.go b/test/framework/cluster.go index 7c6af8013501..386db42a1fdd 100644 --- a/test/framework/cluster.go +++ b/test/framework/cluster.go @@ -106,7 +106,7 @@ func NewClusterE2ETest(t *testing.T, provider Provider, opts ...ClusterE2ETestOp return e } -func WithHardware(requiredCount int) ClusterE2ETestOpt { +func withHardware(requiredCount int, hardareType string, labels map[string]string) ClusterE2ETestOpt { return func(e *ClusterE2ETest) { hardwarePool := e.GetHardwarePool() @@ -118,6 +118,7 @@ func WithHardware(requiredCount int) ClusterE2ETestOpt { for id, h := range hardwarePool { if _, exists := e.TestHardware[id]; !exists { count++ + h.Labels = labels e.TestHardware[id] = h } @@ -127,11 +128,23 @@ func WithHardware(requiredCount int) ClusterE2ETestOpt { } if count < requiredCount { - e.T.Errorf("this test requires at least %d piece(s) of hardware", requiredCount) + e.T.Errorf("this test requires at least %d piece(s) of %s hardware", requiredCount, hardareType) } } } +func WithControlPlaneHardware(requiredCount int) ClusterE2ETestOpt { + return withHardware(requiredCount, api.ControlPlane, map[string]string{api.HardwareLabelTypeKeyName: api.ControlPlane}) +} + +func WithWorkerHardware(requiredCount int) ClusterE2ETestOpt { + return withHardware(requiredCount, api.Worker, map[string]string{api.HardwareLabelTypeKeyName: api.Worker}) +} + +func WithExternalEtcdHardware(requiredCount int) ClusterE2ETestOpt { + return withHardware(requiredCount, api.ExternalEtcd, map[string]string{api.HardwareLabelTypeKeyName: api.ExternalEtcd}) +} + func (e *ClusterE2ETest) GetHardwarePool() map[string]*api.Hardware { if e.HardwarePool == nil { csvFilePath := os.Getenv(tinkerbellInventoryCsvFilePathEnvVar) @@ -253,32 +266,6 @@ func (e *ClusterE2ETest) PowerOffHardware() { } } -func (e *ClusterE2ETest) PXEBootHardware() { - // Initializing BMC Client - ctx := context.Background() - bmcClientFactory := rctrl.NewBMCClientFactoryFunc(ctx) - - for _, h := range e.TestHardware { - bmcClient, err := bmcClientFactory(ctx, h.BMCIPAddress, "623", h.BMCUsername, h.BMCPassword) - if err != nil { - e.T.Fatalf("failed to create bmc client: %v", err) - } - - defer func() { - // Close BMC connection after reconcilation - err = bmcClient.Close(ctx) - if err != nil { - e.T.Fatalf("BMC close connection failed: %v", err) - } - }() - - _, err = bmcClient.SetBootDevice(ctx, string(rapi.PXE), false, true) - if err != nil { - e.T.Fatalf("failed to set boot device to PXE on hardware: %v", err) - } - } -} - func (e *ClusterE2ETest) PowerOnHardware() { // Initializing BMC Client ctx := context.Background() @@ -328,16 +315,17 @@ func (e *ClusterE2ETest) ValidateHardwareDecommissioned() { powerState, err := bmcClient.GetPowerState(ctx) // add sleep retries to give the machine time to power off timeout := 15 - for powerState != string(rapi.Off) && timeout > 0 { + for strings.EqualFold(powerState, string(rapi.Off)) && timeout > 0 { if err != nil { e.T.Logf("failed to get power state for hardware (%v): %v", h, err) } time.Sleep(5 * time.Second) timeout = timeout - 5 powerState, err = bmcClient.GetPowerState(ctx) + e.T.Logf("hardware power state (id=%s, hostname=%s, bmc_ip=%s): power_state=%s", h.MACAddress, h.Hostname, h.BMCIPAddress, powerState) } - if powerState != string(rapi.Off) { + if !strings.EqualFold(powerState, string(rapi.Off)) { e.T.Logf("failed to decommission hardware: id=%s, hostname=%s, bmc_ip=%s", h.MACAddress, h.Hostname, h.BMCIPAddress) failedToDecomm = append(failedToDecomm, h) } else { @@ -441,6 +429,7 @@ func (e *ClusterE2ETest) createCluster(opts ...CommandOpt) { if e.Provider.Name() == TinkerbellProviderName { createClusterArgs = append(createClusterArgs, "-z", e.HardwareCsvLocation) tinkBootstrapIP := os.Getenv(tinkerbellBootstrapIPEnvVar) + e.T.Logf("tinkBootstrapIP: %s", tinkBootstrapIP) if tinkBootstrapIP != "" { createClusterArgs = append(createClusterArgs, "--tinkerbell-bootstrap-ip", tinkBootstrapIP) } diff --git a/test/framework/tinkerbell.go b/test/framework/tinkerbell.go index 714a33276040..eba4afbef22c 100644 --- a/test/framework/tinkerbell.go +++ b/test/framework/tinkerbell.go @@ -59,6 +59,7 @@ func NewTinkerbell(t *testing.T, opts ...TinkerbellOpt) *Tinkerbell { fillers: []api.TinkerbellFiller{ api.WithTinkerbellServer(serverIP), api.WithStringFromEnvVarTinkerbell(tinkerbellSSHAuthorizedKey, api.WithSSHAuthorizedKeyForAllTinkerbellMachines), + api.WithHardwareSelectorLabels(), }, } From 10d24752797c3a5a1de24dd078165cf77df051ed Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Tue, 14 Jun 2022 13:11:55 -0400 Subject: [PATCH 15/27] add br test for tink --- test/e2e/simpleflow_test.go | 11 +++++++++++ test/framework/tinkerbell.go | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/test/e2e/simpleflow_test.go b/test/e2e/simpleflow_test.go index 9e1bb548593d..fb9cb64ec267 100644 --- a/test/e2e/simpleflow_test.go +++ b/test/e2e/simpleflow_test.go @@ -255,6 +255,17 @@ func TestTinkerbellKubernetes123SimpleFlow(t *testing.T) { runTinkerbellSimpleFlow(test) } +func TestTinkerbellKubernetes122BottleRocketSimpleFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithBottleRocketTinkerbell()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube122)), + framework.WithControlPlaneHardware(1), + framework.WithWorkerHardware(1), + ) + runTinkerbellSimpleFlow(test) +} + func TestTinkerbellKubernetes121ExternalEtcdSimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, diff --git a/test/framework/tinkerbell.go b/test/framework/tinkerbell.go index eba4afbef22c..cc76a4385840 100644 --- a/test/framework/tinkerbell.go +++ b/test/framework/tinkerbell.go @@ -143,6 +143,14 @@ func WithUbuntu123Tinkerbell() TinkerbellOpt { } } +func WithBottleRocketTinkerbell() TinkerbellOpt { + return func(t *Tinkerbell) { + t.fillers = append(t.fillers, + api.WithOsFamilyForAllTinkerbellMachines(anywherev1.Bottlerocket), + ) + } +} + func WithTinkerbellExternalEtcdTopology(count int) TinkerbellOpt { return func(t *Tinkerbell) { t.fillers = append([]api.TinkerbellFiller{api.WithTinkerbellEtcdMachineConfig()}, t.fillers...) From 168f681145f8397e3297298c3332a977467fba45 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Tue, 14 Jun 2022 15:21:28 -0400 Subject: [PATCH 16/27] enable k8 1.23 test for tink --- cmd/integration_test/build/buildspecs/test-eks-a-cli.yml | 2 +- test/e2e/simpleflow_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml index 8244fc669540..86530f9feaf7 100644 --- a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml @@ -5,7 +5,7 @@ env: INTEGRATION_TEST_MAX_EC2_COUNT: 130 INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 130 T_CLOUDSTACK_CIDR: "10.11.255.0/24" - SKIPPED_TESTS: "TestVSphereKubernetes122BottlerocketAutoimport,TestCloudStackUpgradeMulticlusterWorkloadClusterWithFluxLegacy,TestCloudStackKubernetes121StackedEtcdRedhat,TestCloudStackKubernetes120RedhatWorkerNodeUpgrade,TestCloudStackKubernetes120RedhatTo121MultipleFieldsUpgrade,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC,TestSnowKubernetes121UbuntuProxyConfig" + SKIPPED_TESTS: "TestTinkerbellKubernetes122BottleRocketSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdSimpleFlow,TestTinkerbellKubernetes123SimpleFlow,TestVSphereKubernetes122BottlerocketAutoimport,TestCloudStackUpgradeMulticlusterWorkloadClusterWithFluxLegacy,TestCloudStackKubernetes121StackedEtcdRedhat,TestCloudStackKubernetes120RedhatWorkerNodeUpgrade,TestCloudStackKubernetes120RedhatTo121MultipleFieldsUpgrade,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC,TestSnowKubernetes121UbuntuProxyConfig" CLOUDSTACK_PROVIDER: true EKSA_GIT_KNOWN_HOSTS: "/tmp/known_hosts" EKSA_GIT_PRIVATE_KEY: "/tmp/private-key" diff --git a/test/e2e/simpleflow_test.go b/test/e2e/simpleflow_test.go index fb9cb64ec267..ba4300a63359 100644 --- a/test/e2e/simpleflow_test.go +++ b/test/e2e/simpleflow_test.go @@ -251,6 +251,7 @@ func TestTinkerbellKubernetes123SimpleFlow(t *testing.T) { framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube123)), framework.WithControlPlaneHardware(1), framework.WithWorkerHardware(1), + framework.WithEnvVar("K8S_1_23_SUPPORT", "true"), ) runTinkerbellSimpleFlow(test) } From f7b9fe943928914f54524ae9b7e87c13a4b5e030 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Wed, 15 Jun 2022 13:18:58 -0400 Subject: [PATCH 17/27] update conformance and prow tests to accound for tinkerbell changes --- .../buildspecs/conformance-eks-a-cli.yml | 27 ++++++++-- .../build/buildspecs/test-eks-a-cli.yml | 7 ++- internal/pkg/vsphere/deploy.go | 2 +- internal/test/e2e/run.go | 12 +++-- internal/test/e2e/testRunner.go | 2 +- internal/test/e2e/tinkerbell.go | 13 ++--- pkg/executables/govc.go | 1 - scripts/e2e_test_docker.sh | 15 +++++- test/e2e/conformance_test.go | 50 ++++++++++++++++++- 9 files changed, 107 insertions(+), 22 deletions(-) diff --git a/cmd/integration_test/build/buildspecs/conformance-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/conformance-eks-a-cli.yml index b9b91fb97d6e..3b9de988c04f 100644 --- a/cmd/integration_test/build/buildspecs/conformance-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/conformance-eks-a-cli.yml @@ -4,6 +4,12 @@ env: variables: INTEGRATION_TEST_MAX_EC2_COUNT: 25 INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 25 + T_TINKERBELL_MAX_HARDWARE_PER_TEST: 5 + T_TINKERBELL_INVENTORY_CSV: "hardware-manifests/inventory.csv" + T_TINKERBELL_BOOTSTRAP_INTERFACE: "ens192" + TEST_RUNNER_GOVC_LIBRARY: "eks-a-templates" + TEST_RUNNER_GOVC_TEMPLATE: "eks-a-admin-ci" + INTEGRATION_TEST_INFRA_CONFIG: "/tmp/test-infra.yml" secrets-manager: EKSA_VSPHERE_USERNAME: "vsphere_ci_beta_connection:vsphere_username" EKSA_VSPHERE_PASSWORD: "vsphere_ci_beta_connection:vsphere_password" @@ -33,11 +39,27 @@ env: T_VSPHERE_TEMPLATE_BR_1_23: "vsphere_ci_beta_connection:template_br_23" T_VSPHERE_TLS_INSECURE: "vsphere_ci_beta_connection:tls_insecure" T_VSPHERE_TLS_THUMBPRINT: "vsphere_ci_beta_connection:tls_thumbprint" + T_TINKERBELL_IMAGE_UBUNTU_1_20: "tinkerbell_ci:image_ubuntu_1_20" + T_TINKERBELL_IMAGE_UBUNTU_1_21: "tinkerbell_ci:image_ubuntu_1_21" + T_TINKERBELL_IMAGE_UBUNTU_1_22: "tinkerbell_ci:image_ubuntu_1_22" + T_TINKERBELL_IMAGE_UBUNTU_1_23: "tinkerbell_ci:image_ubuntu_1_23" + T_TINKERBELL_SSH_AUTHORIZED_KEY: "vsphere_ci_beta_connection:ssh_authorized_key" + T_TINKERBELL_NETWORK_CIDR: "tinkerbell_ci:network_cidr" + T_TINKERBELL_S3_INVENTORY_CSV_KEY: "tinkerbell_ci:s3_inventory_csv" + TEST_RUNNER_GOVC_USERNAME: "tinkerbell_ci:govc_username" + TEST_RUNNER_GOVC_PASSWORD: "tinkerbell_ci: govc_password" + TEST_RUNNER_GOVC_URL: "tinkerbell_ci: govc_url" + TEST_RUNNER_GOVC_DATACENTER: "tinkerbell_ci: govc_datacenter" + TEST_RUNNER_GOVC_DATASTORE: "tinkerbell_ci: govc_datastore" + TEST_RUNNER_GOVC_RESOURCE_POOL: "tinkerbell_ci: govc_resource_pool" + TEST_RUNNER_GOVC_NETWORK: "tinkerbell_ci: govc_network" + TEST_RUNNER_GOVC_FOLDER: "tinkerbell_ci: govc_folder" phases: pre_build: commands: - source ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/setup_profile.sh + - source ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/create_infra_config.sh - ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/start_docker.sh - make conformance-tests build: @@ -45,12 +67,11 @@ phases: - export JOB_ID=$CODEBUILD_BUILD_ID - > ./bin/test e2e run - -a ${INTEGRATION_TEST_AL2_AMI_ID} + -c ${INTEGRATION_TEST_INFRA_CONFIG} -s ${INTEGRATION_TEST_STORAGE_BUCKET} -j ${JOB_ID} -i ${INTEGRATION_TEST_INSTANCE_PROFILE} - -n ${INTEGRATION_TEST_SUBNET_ID} -m ${INTEGRATION_TEST_MAX_EC2_COUNT} - -c ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT} + -p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT} -r 'Test' --cleanup-vms=true \ No newline at end of file diff --git a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml index 86530f9feaf7..d135ba0b31dc 100644 --- a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml @@ -12,6 +12,8 @@ env: T_TINKERBELL_MAX_HARDWARE_PER_TEST: 4 T_TINKERBELL_INVENTORY_CSV: "hardware-manifests/inventory.csv" T_TINKERBELL_BOOTSTRAP_INTERFACE: "ens192" + TEST_RUNNER_GOVC_LIBRARY: "eks-a-templates" + TEST_RUNNER_GOVC_TEMPLATE: "eks-a-admin-ci" INTEGRATION_TEST_INFRA_CONFIG: "/tmp/test-infra.yml" secrets-manager: EKSA_VSPHERE_USERNAME: "vsphere_ci_beta_connection:vsphere_username" @@ -73,21 +75,18 @@ env: T_TINKERBELL_IMAGE_UBUNTU_1_21: "tinkerbell_ci:image_ubuntu_1_21" T_TINKERBELL_IMAGE_UBUNTU_1_22: "tinkerbell_ci:image_ubuntu_1_22" T_TINKERBELL_IMAGE_UBUNTU_1_23: "tinkerbell_ci:image_ubuntu_1_23" - T_TINKERBELL_SSH_AUTHORIZED_KEY: "tinkerbell_ci:ssh_key" + T_TINKERBELL_SSH_AUTHORIZED_KEY: "vsphere_ci_beta_connection:ssh_authorized_key" T_TINKERBELL_NETWORK_CIDR: "tinkerbell_ci:network_cidr" T_TINKERBELL_S3_INVENTORY_CSV_KEY: "tinkerbell_ci:s3_inventory_csv" TEST_RUNNER_GOVC_USERNAME: "tinkerbell_ci:govc_username" TEST_RUNNER_GOVC_PASSWORD: "tinkerbell_ci: govc_password" TEST_RUNNER_GOVC_URL: "tinkerbell_ci: govc_url" - TEST_RUNNER_GOVC_LIBRARY: "tinkerbell_ci: govc_library" - TEST_RUNNER_GOVC_TEMPLATE: "tinkerbell_ci: govc_template" TEST_RUNNER_GOVC_DATACENTER: "tinkerbell_ci: govc_datacenter" TEST_RUNNER_GOVC_DATASTORE: "tinkerbell_ci: govc_datastore" TEST_RUNNER_GOVC_RESOURCE_POOL: "tinkerbell_ci: govc_resource_pool" TEST_RUNNER_GOVC_NETWORK: "tinkerbell_ci: govc_network" TEST_RUNNER_GOVC_FOLDER: "tinkerbell_ci: govc_folder" - phases: pre_build: commands: diff --git a/internal/pkg/vsphere/deploy.go b/internal/pkg/vsphere/deploy.go index cd2f57fb8446..0a43062c0d95 100644 --- a/internal/pkg/vsphere/deploy.go +++ b/internal/pkg/vsphere/deploy.go @@ -48,7 +48,7 @@ func DeployTemplate(library, templateName, vmName, deployFolder, datacenter, dat } // deploy template - if err := govc.DeployTemplate(context, library, templateName, vmName, deployFolder, datacenter, datastore, resourcePool, deployOptions); err != nil { + if err := govc.DeployTemplate(context, library, templateName, vmName, deployFolder, datacenter, datastore, opts.NetworkMappings[0].Network, resourcePool, deployOptions); err != nil { return fmt.Errorf("failed to deploy vm from library template: %v", err) } diff --git a/internal/test/e2e/run.go b/internal/test/e2e/run.go index 9f8b0ecdac33..f36cbac38799 100644 --- a/internal/test/e2e/run.go +++ b/internal/test/e2e/run.go @@ -244,7 +244,7 @@ func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, er awsSession, err := session.NewSession() if err != nil { - return nil, fmt.Errorf("creating aws session for test: %v", err) + return nil, fmt.Errorf("creating aws session for tests: %v", err) } testRunnerConfig, err := NewTestRunnerConfigFromFile(conf.TestInstanceConfigFile) @@ -292,9 +292,13 @@ func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, er return nil, fmt.Errorf("failed to get Tinkerbell hardware: %v", err) } - maxHardwarePerE2ETest, err := strconv.Atoi(os.Getenv(MaxHardwarePerE2ETestEnvVar)) - if err != nil { - return nil, fmt.Errorf("failed to get Tinkerbell max hardware per test env var: %v", err) + maxHardwarePerE2ETest := TinkerbellDefaultMaxHardwarePerE2ETest + maxHardwareEnvValue := os.Getenv(MaxHardwarePerE2ETestEnvVar) + if len(maxHardwareEnvValue) > 0 { + maxHardwarePerE2ETest, err = strconv.Atoi(maxHardwareEnvValue) + if err != nil { + return nil, fmt.Errorf("failed to get Tinkerbell max hardware per test env var: %v", err) + } } logger.V(1).Info("INFO:", "totalHardware", len(hardware)) diff --git a/internal/test/e2e/testRunner.go b/internal/test/e2e/testRunner.go index 880412973c0b..0a8800e3f82c 100644 --- a/internal/test/e2e/testRunner.go +++ b/internal/test/e2e/testRunner.go @@ -200,7 +200,7 @@ func (v *VSphereTestRunner) tagInstance(c instanceRunConf, key, value string) er } func (e *Ec2TestRunner) tagInstance(c instanceRunConf, key, value string) error { - err := ec2.TagInstance(c.session, c.instanceId, key, value) + err := ec2.TagInstance(c.session, e.InstanceID, key, value) if err != nil { return fmt.Errorf("failed to tag Ec2 test runner: %v", err) } diff --git a/internal/test/e2e/tinkerbell.go b/internal/test/e2e/tinkerbell.go index 07767ed0503e..dda051a435ef 100644 --- a/internal/test/e2e/tinkerbell.go +++ b/internal/test/e2e/tinkerbell.go @@ -12,12 +12,13 @@ import ( ) const ( - tinkerbellInventoryCsvFilePathEnvVar = "T_TINKERBELL_INVENTORY_CSV" - tinkerbellHardwareS3FileKeyEnvVar = "T_TINKERBELL_S3_INVENTORY_CSV_KEY" - tinkerbellTestsRe = `^.*Tinkerbell.*$` - e2eHardwareCsvFilePath = "e2e-inventory.csv" - MaxHardwarePerE2ETestEnvVar = "T_TINKERBELL_MAX_HARDWARE_PER_TEST" - tinkerbellBootstrapInterfaceEnvVar = "T_TINKERBELL_BOOTSTRAP_INTERFACE" + tinkerbellInventoryCsvFilePathEnvVar = "T_TINKERBELL_INVENTORY_CSV" + tinkerbellHardwareS3FileKeyEnvVar = "T_TINKERBELL_S3_INVENTORY_CSV_KEY" + tinkerbellTestsRe = `^.*Tinkerbell.*$` + e2eHardwareCsvFilePath = "e2e-inventory.csv" + MaxHardwarePerE2ETestEnvVar = "T_TINKERBELL_MAX_HARDWARE_PER_TEST" + TinkerbellDefaultMaxHardwarePerE2ETest = 4 + tinkerbellBootstrapInterfaceEnvVar = "T_TINKERBELL_BOOTSTRAP_INTERFACE" ) func (e *E2ESession) setupTinkerbellEnv(testRegex string) error { diff --git a/pkg/executables/govc.go b/pkg/executables/govc.go index 72712af6224b..2558ad35bc16 100644 --- a/pkg/executables/govc.go +++ b/pkg/executables/govc.go @@ -372,7 +372,6 @@ func (g *Govc) DeployTemplate(ctx context.Context, library, templateName, vmName } deployOptsPath, err := g.writer.Write(DeployOptsFile, deployOpts, filewriter.PersistentFile) - if err != nil { return fmt.Errorf("failed writing deploy options file to disk: %v", err) } diff --git a/scripts/e2e_test_docker.sh b/scripts/e2e_test_docker.sh index 1a2ab3243486..0519a536991f 100755 --- a/scripts/e2e_test_docker.sh +++ b/scripts/e2e_test_docker.sh @@ -43,6 +43,19 @@ role_arn=$TEST_ROLE_ARN region=${AWS_REGION:-${AWS_DEFAULT_REGION:-us-west-2}} source_profile=default EOF + +INTEGRATION_TEST_INFRA_CONFIG="/tmp/test-infra.yml" +export T_TINKERBELL_S3_INVENTORY_CSV_KEY="inventory/den80/den80-hardware.csv" + +cat << EOF > ${INTEGRATION_TEST_INFRA_CONFIG} +--- + +ec2: + amiId: ${INTEGRATION_TEST_AL2_AMI_ID} + subnetId: + +EOF + export AWS_SDK_LOAD_CONFIG=true export AWS_CONFIG_FILE=$(pwd)/config_file export AWS_PROFILE=e2e-docker-test @@ -58,7 +71,7 @@ fi TEST_REPORT_FOLDER=/logs/artifacts $BIN_FOLDER/test e2e run \ - -a ${INTEGRATION_TEST_AL2_AMI_ID} \ + -c ${INTEGRATION_TEST_INFRA_CONFIG} \ -s ${INTEGRATION_TEST_STORAGE_BUCKET} \ -j ${JOB_ID} \ -i ${INTEGRATION_TEST_INSTANCE_PROFILE} \ diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go index 32438f8d5489..81b14d4406b7 100644 --- a/test/e2e/conformance_test.go +++ b/test/e2e/conformance_test.go @@ -19,6 +19,15 @@ func runConformanceFlow(test *framework.ClusterE2ETest) { test.DeleteCluster() } +func runTinkerbellConformanceFlow(test *framework.ClusterE2ETest) { + test.GenerateClusterConfig() + test.GenerateHardwareConfig() + test.PowerOffHardware() + test.CreateCluster() + test.DeleteCluster() + test.ValidateHardwareDecommissioned() +} + func TestDockerKubernetes120ThreeWorkersConformanceFlow(t *testing.T) { test := framework.NewClusterE2ETest( t, @@ -160,4 +169,43 @@ func TestCloudStackKubernetes121ThreeWorkersConformanceFlow(t *testing.T) { framework.WithClusterFiller(api.WithWorkerNodeCount(3)), ) runConformanceFlow(test) -} \ No newline at end of file +} + +func TestTinkerbellKubernetes120ThreeReplicasTwoWorkersConformanceFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithUbuntu120Tinkerbell()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube120)), + framework.WithClusterFiller(api.WithWorkerNodeCount(2)), + framework.WithClusterFiller(api.WithControlPlaneCount(3)), + framework.WithControlPlaneHardware(3), + framework.WithWorkerHardware(2), + ) + runTinkerbellConformanceFlow(test) +} + +func TestTinkerbellKubernetes121ThreeReplicasTwoWorkersConformanceFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithUbuntu121Tinkerbell()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube121)), + framework.WithClusterFiller(api.WithWorkerNodeCount(2)), + framework.WithClusterFiller(api.WithControlPlaneCount(3)), + framework.WithControlPlaneHardware(3), + framework.WithWorkerHardware(2), + ) + runTinkerbellConformanceFlow(test) +} + +func TestTinkerbellKubernetes122ThreeReplicasTwoWorkersConformanceFlow(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewTinkerbell(t, framework.WithUbuntu122Tinkerbell()), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube122)), + framework.WithClusterFiller(api.WithWorkerNodeCount(2)), + framework.WithClusterFiller(api.WithControlPlaneCount(3)), + framework.WithControlPlaneHardware(3), + framework.WithWorkerHardware(2), + ) + runTinkerbellConformanceFlow(test) +} From 4a4c873e7aea203d46e5809cf2123271836492a3 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 16 Jun 2022 08:26:30 -0400 Subject: [PATCH 18/27] split tink and cp cidr ranges --- .../buildspecs/conformance-eks-a-cli.yml | 1 + .../build/buildspecs/test-eks-a-cli.yml | 1 + test/framework/tinkerbell.go | 24 +++++++++++-------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/cmd/integration_test/build/buildspecs/conformance-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/conformance-eks-a-cli.yml index 3b9de988c04f..61a7bb39b6f2 100644 --- a/cmd/integration_test/build/buildspecs/conformance-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/conformance-eks-a-cli.yml @@ -45,6 +45,7 @@ env: T_TINKERBELL_IMAGE_UBUNTU_1_23: "tinkerbell_ci:image_ubuntu_1_23" T_TINKERBELL_SSH_AUTHORIZED_KEY: "vsphere_ci_beta_connection:ssh_authorized_key" T_TINKERBELL_NETWORK_CIDR: "tinkerbell_ci:network_cidr" + T_TINKERBELL_CP_NETWORK_CIDR: "tinkerbell_ci:cp_network_cidr" T_TINKERBELL_S3_INVENTORY_CSV_KEY: "tinkerbell_ci:s3_inventory_csv" TEST_RUNNER_GOVC_USERNAME: "tinkerbell_ci:govc_username" TEST_RUNNER_GOVC_PASSWORD: "tinkerbell_ci: govc_password" diff --git a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml index d135ba0b31dc..4d355029ce49 100644 --- a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml @@ -77,6 +77,7 @@ env: T_TINKERBELL_IMAGE_UBUNTU_1_23: "tinkerbell_ci:image_ubuntu_1_23" T_TINKERBELL_SSH_AUTHORIZED_KEY: "vsphere_ci_beta_connection:ssh_authorized_key" T_TINKERBELL_NETWORK_CIDR: "tinkerbell_ci:network_cidr" + T_TINKERBELL_CP_NETWORK_CIDR: "tinkerbell_ci:cp_network_cidr" T_TINKERBELL_S3_INVENTORY_CSV_KEY: "tinkerbell_ci:s3_inventory_csv" TEST_RUNNER_GOVC_USERNAME: "tinkerbell_ci:govc_username" TEST_RUNNER_GOVC_PASSWORD: "tinkerbell_ci: govc_password" diff --git a/test/framework/tinkerbell.go b/test/framework/tinkerbell.go index cc76a4385840..c68168a6009f 100644 --- a/test/framework/tinkerbell.go +++ b/test/framework/tinkerbell.go @@ -9,19 +9,21 @@ import ( ) const ( - TinkerbellProviderName = "tinkerbell" - tinkerbellBootstrapIPEnvVar = "T_TINKERBELL_BOOTSTRAP_IP" - tinkerbellNetworkCidrEnvVar = "T_TINKERBELL_NETWORK_CIDR" - tinkerbellImageUbuntu120EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_20" - tinkerbellImageUbuntu121EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_21" - tinkerbellImageUbuntu122EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_22" - tinkerbellImageUbuntu123EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_23" - tinkerbellInventoryCsvFilePathEnvVar = "T_TINKERBELL_INVENTORY_CSV" - tinkerbellSSHAuthorizedKey = "T_TINKERBELL_SSH_AUTHORIZED_KEY" + TinkerbellProviderName = "tinkerbell" + tinkerbellBootstrapIPEnvVar = "T_TINKERBELL_BOOTSTRAP_IP" + tinkerbellNetworkCidrEnvVar = "T_TINKERBELL_NETWORK_CIDR" + tinkerbellControlPlaneNetworkCidrEnvVar = "T_TINKERBELL_CP_NETWORK_CIDR" + tinkerbellImageUbuntu120EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_20" + tinkerbellImageUbuntu121EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_21" + tinkerbellImageUbuntu122EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_22" + tinkerbellImageUbuntu123EnvVar = "T_TINKERBELL_IMAGE_UBUNTU_1_23" + tinkerbellInventoryCsvFilePathEnvVar = "T_TINKERBELL_INVENTORY_CSV" + tinkerbellSSHAuthorizedKey = "T_TINKERBELL_SSH_AUTHORIZED_KEY" ) var requiredTinkerbellEnvVars = []string{ tinkerbellNetworkCidrEnvVar, + tinkerbellControlPlaneNetworkCidrEnvVar, tinkerbellImageUbuntu120EnvVar, tinkerbellImageUbuntu121EnvVar, tinkerbellImageUbuntu122EnvVar, @@ -42,6 +44,7 @@ type Tinkerbell struct { clusterFillers []api.ClusterFiller serverIP string cidr string + controlPlaneCidr string inventoryCsvFilePath string } @@ -66,6 +69,7 @@ func NewTinkerbell(t *testing.T, opts ...TinkerbellOpt) *Tinkerbell { tink.serverIP = serverIP tink.cidr = cidr + tink.controlPlaneCidr = os.Getenv(tinkerbellControlPlaneNetworkCidrEnvVar) tink.inventoryCsvFilePath = os.Getenv(tinkerbellInventoryCsvFilePathEnvVar) for _, opt := range opts { @@ -98,7 +102,7 @@ func (t *Tinkerbell) customizeProviderConfig(file string, fillers ...api.Tinkerb } func (t *Tinkerbell) ClusterConfigFillers() []api.ClusterFiller { - clusterIP, err := GenerateUniqueIp(t.cidr) + clusterIP, err := GenerateUniqueIp(t.controlPlaneCidr) if err != nil { t.t.Fatalf("failed to generate cluster ip from cidr %s: %v", t.cidr, err) } From cf1194c8a8630df5b387e6902e54c736b7935239 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 16 Jun 2022 10:35:45 -0400 Subject: [PATCH 19/27] persist vm on failure for debug --- internal/test/e2e/run.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/test/e2e/run.go b/internal/test/e2e/run.go index f36cbac38799..e8e9525c0fae 100644 --- a/internal/test/e2e/run.go +++ b/internal/test/e2e/run.go @@ -136,12 +136,17 @@ func RunTests(conf instanceRunConf) (testInstanceID string, testCommandResult *t return "", nil, err } - defer func() { - err := testRunner.decommInstance(conf) - if err != nil { - logger.V(1).Info("WARN: Failed to decomm e2e test runner instance", "error", err) + success := false + defer func(successRef *bool) { + if *successRef { + err := testRunner.decommInstance(conf) + if err != nil { + logger.V(1).Info("WARN: Failed to decomm e2e test runner instance", "error", err) + } + } else { + logger.V(1).Info("Persist instance on failure", "instance", instanceId) } - }() + }(&success) session, err := newE2ESession(instanceId, conf) if err != nil { @@ -158,6 +163,8 @@ func RunTests(conf instanceRunConf) (testInstanceID string, testCommandResult *t return session.instanceId, nil, err } + success = testCommandResult.Successful() + if err = conf.runPostTestsProcessing(session, testCommandResult); err != nil { return session.instanceId, nil, err } From 36155aa45097d6f4f096fcd1fbaf72deb8d90db1 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 16 Jun 2022 12:06:02 -0400 Subject: [PATCH 20/27] fix bundle upload --- internal/test/e2e/artifacts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/test/e2e/artifacts.go b/internal/test/e2e/artifacts.go index 22eafdbf11a5..67a6f86b4dcb 100644 --- a/internal/test/e2e/artifacts.go +++ b/internal/test/e2e/artifacts.go @@ -29,7 +29,7 @@ func (e *E2ESession) uploadGeneratedFilesFromInstance(testName string) { func (e *E2ESession) uploadDiagnosticArchiveFromInstance(testName string) { bundleNameFormat := "support-bundle-*.tar.gz" logger.V(1).Info("Uploading diagnostic bundle to s3 bucket") - command := newCopyCommand().from(e2eHomeFolder, clusterName(e.branchName, e.instanceId, testName)).to( + command := newCopyCommand().from(e2eHomeFolder).to( e.generatedArtifactsBucketPath(), testName, ).recursive().exclude("*").include(bundleNameFormat).String() From 170acc48d1ef97a28ddc1086513e1f7c4e50b65d Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 16 Jun 2022 12:19:36 -0400 Subject: [PATCH 21/27] update tink conformance tests --- test/e2e/conformance_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go index 81b14d4406b7..3d1f6c16ba20 100644 --- a/test/e2e/conformance_test.go +++ b/test/e2e/conformance_test.go @@ -8,6 +8,7 @@ import ( "github.com/aws/eks-anywhere/internal/pkg/api" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + "github.com/aws/eks-anywhere/pkg/features" "github.com/aws/eks-anywhere/test/framework" ) @@ -24,6 +25,8 @@ func runTinkerbellConformanceFlow(test *framework.ClusterE2ETest) { test.GenerateHardwareConfig() test.PowerOffHardware() test.CreateCluster() + test.RunConformanceTests() + test.StopIfFailed() test.DeleteCluster() test.ValidateHardwareDecommissioned() } From e3114530c7d7641f2e9e681d56f829c1f05ea509 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 16 Jun 2022 13:23:10 -0400 Subject: [PATCH 22/27] remove example infra config --- .../build/buildspecs/test-infra.yml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 cmd/integration_test/build/buildspecs/test-infra.yml diff --git a/cmd/integration_test/build/buildspecs/test-infra.yml b/cmd/integration_test/build/buildspecs/test-infra.yml deleted file mode 100644 index 9730c275022f..000000000000 --- a/cmd/integration_test/build/buildspecs/test-infra.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- - -ec2: - amiId: ami-00c71c2441b729da0 - subnetId: subnet-0fd4b6ba7c9e17b5f - -vSphere: - url: https://10.80.148.25 - insecure: True - library: eks-a-templates - template: eks-a-admin-ci - datacenter: Denver - datastore: datastore1.2 - resourcePool: CI - network: CI - folder: CI \ No newline at end of file From 5976d06924425a2db17179e4d08e74edc6bc1d8b Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 16 Jun 2022 15:22:20 -0400 Subject: [PATCH 23/27] skip three worker test --- cmd/integration_test/build/buildspecs/test-eks-a-cli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml index 4d355029ce49..60ba9ee6a104 100644 --- a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml @@ -5,7 +5,7 @@ env: INTEGRATION_TEST_MAX_EC2_COUNT: 130 INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 130 T_CLOUDSTACK_CIDR: "10.11.255.0/24" - SKIPPED_TESTS: "TestTinkerbellKubernetes122BottleRocketSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdSimpleFlow,TestTinkerbellKubernetes123SimpleFlow,TestVSphereKubernetes122BottlerocketAutoimport,TestCloudStackUpgradeMulticlusterWorkloadClusterWithFluxLegacy,TestCloudStackKubernetes121StackedEtcdRedhat,TestCloudStackKubernetes120RedhatWorkerNodeUpgrade,TestCloudStackKubernetes120RedhatTo121MultipleFieldsUpgrade,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC,TestSnowKubernetes121UbuntuProxyConfig" + SKIPPED_TESTS: "TestTinkerbellKubernetes121ThreeWorkersSimpleFlow,TestTinkerbellKubernetes122BottleRocketSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdSimpleFlow,TestTinkerbellKubernetes123SimpleFlow,TestVSphereKubernetes122BottlerocketAutoimport,TestCloudStackUpgradeMulticlusterWorkloadClusterWithFluxLegacy,TestCloudStackKubernetes121StackedEtcdRedhat,TestCloudStackKubernetes120RedhatWorkerNodeUpgrade,TestCloudStackKubernetes120RedhatTo121MultipleFieldsUpgrade,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC,TestSnowKubernetes121UbuntuProxyConfig" CLOUDSTACK_PROVIDER: true EKSA_GIT_KNOWN_HOSTS: "/tmp/known_hosts" EKSA_GIT_PRIVATE_KEY: "/tmp/private-key" From 8a79c8b083c5e3caf771c296628826a5b5fe0f8e Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 16 Jun 2022 15:59:09 -0400 Subject: [PATCH 24/27] add ping test to show connectivity to vcenter in colo lab thru direct connect --- scripts/e2e_test_docker.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/e2e_test_docker.sh b/scripts/e2e_test_docker.sh index 0519a536991f..2dfa3d986651 100755 --- a/scripts/e2e_test_docker.sh +++ b/scripts/e2e_test_docker.sh @@ -66,6 +66,9 @@ if [ -f "$BIN_FOLDER/local-bundle-release.yaml" ]; then BUNDLES_OVERRIDE=true fi +echo "Ping vCenter in EKS-A: " +ping -c 1 10.80.148.25 &> /dev/null && echo success || echo fail + # In order to be uploaded from the sidecar and used by the junit lens # the junit reports need to be in /logs/*/junit*.xml TEST_REPORT_FOLDER=/logs/artifacts @@ -92,4 +95,6 @@ $REPO_ROOT/cmd/integration_test/build/script/upload_artifacts.sh \ $PULL_PULL_SHA \ "linux" \ "amd64" \ - true \ No newline at end of file + true + + ping -c 1 10.80.148.25 &> /dev/null && echo success || echo fail \ No newline at end of file From bc85293e01c2e07f0269275bdafd47161e3bcfb0 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 16 Jun 2022 16:06:58 -0400 Subject: [PATCH 25/27] remove ping --- scripts/e2e_test_docker.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/e2e_test_docker.sh b/scripts/e2e_test_docker.sh index 2dfa3d986651..f5f990bf2d16 100755 --- a/scripts/e2e_test_docker.sh +++ b/scripts/e2e_test_docker.sh @@ -66,9 +66,6 @@ if [ -f "$BIN_FOLDER/local-bundle-release.yaml" ]; then BUNDLES_OVERRIDE=true fi -echo "Ping vCenter in EKS-A: " -ping -c 1 10.80.148.25 &> /dev/null && echo success || echo fail - # In order to be uploaded from the sidecar and used by the junit lens # the junit reports need to be in /logs/*/junit*.xml TEST_REPORT_FOLDER=/logs/artifacts From b57382239041a93d553a219952791d82bfd6f0ce Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Thu, 16 Jun 2022 19:45:07 -0400 Subject: [PATCH 26/27] only run tink tests on main branch --- internal/test/e2e/run.go | 17 +++++++++++++++-- test/framework/cluster.go | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/internal/test/e2e/run.go b/internal/test/e2e/run.go index e8e9525c0fae..715ee3289c20 100644 --- a/internal/test/e2e/run.go +++ b/internal/test/e2e/run.go @@ -164,6 +164,7 @@ func RunTests(conf instanceRunConf) (testInstanceID string, testCommandResult *t } success = testCommandResult.Successful() + logger.V(1).Info("Instance Test Result", "instance", instanceId, "success", success) if err = conf.runPostTestsProcessing(session, testCommandResult); err != nil { return session.instanceId, nil, err @@ -236,6 +237,8 @@ func (e *E2ESession) commandWithEnvVars(command string) string { } func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, error) { + var runConfs []instanceRunConf + testPerInstance := len(testsList) / conf.MaxInstances if testPerInstance == 0 { testPerInstance = 1 @@ -246,7 +249,6 @@ func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, er privateNetworkTestsRe := regexp.MustCompile(`^.*(Proxy|RegistryMirror).*$`) multiClusterTestsRe := regexp.MustCompile(`^.*Multicluster.*$`) - runConfs := make([]instanceRunConf, 0, conf.MaxInstances) ipman := newE2EIPManager(os.Getenv(cidrVar), os.Getenv(privateNetworkCidrVar)) awsSession, err := session.NewSession() @@ -289,7 +291,18 @@ func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, er } } - err = s3.DownloadToDisk(awsSession, os.Getenv(tinkerbellHardwareS3FileKeyEnvVar), conf.StorageBucket, e2eHardwareCsvFilePath) + if strings.EqualFold(conf.BranchName, "main") { + runConfs, err = splitTinkerbellTests(awsSession, testsList, conf, testRunnerConfig, runConfs) + if err != nil { + return nil, fmt.Errorf("failed to split Tinkerbell tests: %v", err) + } + } + + return runConfs, nil +} + +func splitTinkerbellTests(awsSession *session.Session, testsList []string, conf ParallelRunConf, testRunnerConfig *TestInfraConfig, runConfs []instanceRunConf) ([]instanceRunConf, error) { + err := s3.DownloadToDisk(awsSession, os.Getenv(tinkerbellHardwareS3FileKeyEnvVar), conf.StorageBucket, e2eHardwareCsvFilePath) if err != nil { return nil, fmt.Errorf("failed to download tinkerbell hardware csv: %v", err) } diff --git a/test/framework/cluster.go b/test/framework/cluster.go index 386db42a1fdd..1006e8640486 100644 --- a/test/framework/cluster.go +++ b/test/framework/cluster.go @@ -315,7 +315,7 @@ func (e *ClusterE2ETest) ValidateHardwareDecommissioned() { powerState, err := bmcClient.GetPowerState(ctx) // add sleep retries to give the machine time to power off timeout := 15 - for strings.EqualFold(powerState, string(rapi.Off)) && timeout > 0 { + for !strings.EqualFold(powerState, string(rapi.Off)) && timeout > 0 { if err != nil { e.T.Logf("failed to get power state for hardware (%v): %v", h, err) } From 2392e3923d84ffa417cd259eb8858e0d5f50c9e8 Mon Sep 17 00:00:00 2001 From: Greg Westerfield Date: Fri, 17 Jun 2022 09:24:46 -0400 Subject: [PATCH 27/27] update test concurrent count --- cmd/integration_test/build/buildspecs/test-eks-a-cli.yml | 4 ++-- internal/test/e2e/run.go | 3 +-- scripts/e2e_test_docker.sh | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml index 60ba9ee6a104..b9e723f6a9b8 100644 --- a/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/test-eks-a-cli.yml @@ -2,8 +2,8 @@ version: 0.2 env: variables: - INTEGRATION_TEST_MAX_EC2_COUNT: 130 - INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 130 + INTEGRATION_TEST_MAX_EC2_COUNT: 160 + INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 160 T_CLOUDSTACK_CIDR: "10.11.255.0/24" SKIPPED_TESTS: "TestTinkerbellKubernetes121ThreeWorkersSimpleFlow,TestTinkerbellKubernetes122BottleRocketSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdSimpleFlow,TestTinkerbellKubernetes123SimpleFlow,TestVSphereKubernetes122BottlerocketAutoimport,TestCloudStackUpgradeMulticlusterWorkloadClusterWithFluxLegacy,TestCloudStackKubernetes121StackedEtcdRedhat,TestCloudStackKubernetes120RedhatWorkerNodeUpgrade,TestCloudStackKubernetes120RedhatTo121MultipleFieldsUpgrade,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC,TestSnowKubernetes121UbuntuProxyConfig" CLOUDSTACK_PROVIDER: true diff --git a/internal/test/e2e/run.go b/internal/test/e2e/run.go index 715ee3289c20..76f7ecc5bb2b 100644 --- a/internal/test/e2e/run.go +++ b/internal/test/e2e/run.go @@ -237,8 +237,6 @@ func (e *E2ESession) commandWithEnvVars(command string) string { } func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, error) { - var runConfs []instanceRunConf - testPerInstance := len(testsList) / conf.MaxInstances if testPerInstance == 0 { testPerInstance = 1 @@ -249,6 +247,7 @@ func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, er privateNetworkTestsRe := regexp.MustCompile(`^.*(Proxy|RegistryMirror).*$`) multiClusterTestsRe := regexp.MustCompile(`^.*Multicluster.*$`) + runConfs := make([]instanceRunConf, 0, conf.MaxInstances) ipman := newE2EIPManager(os.Getenv(cidrVar), os.Getenv(privateNetworkCidrVar)) awsSession, err := session.NewSession() diff --git a/scripts/e2e_test_docker.sh b/scripts/e2e_test_docker.sh index f5f990bf2d16..6529f71074bd 100755 --- a/scripts/e2e_test_docker.sh +++ b/scripts/e2e_test_docker.sh @@ -93,5 +93,3 @@ $REPO_ROOT/cmd/integration_test/build/script/upload_artifacts.sh \ "linux" \ "amd64" \ true - - ping -c 1 10.80.148.25 &> /dev/null && echo success || echo fail \ No newline at end of file