add tinkerbell ci test harness to e2e test framework

cmd/integration_test/build/buildspecs/test-eks-a-cli.yml

@@ -84,7 +84,7 @@ phases:
         -i ${INTEGRATION_TEST_INSTANCE_PROFILE}
         -n ${INTEGRATION_TEST_SUBNET_ID}
         -m ${INTEGRATION_TEST_MAX_EC2_COUNT}
-        -c ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
+        -p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
         -r 'Test'
         -v 4
         --skip ${SKIPPED_TESTS}

cmd/integration_test/build/buildspecs/test-infra.yml

@@ -0,0 +1,16 @@
+---
+
+ec2:
+  amiId: ami-00c71c2441b729da0
+  subnetId: subnet-0fd4b6ba7c9e17b5f
+
+vSphere:
+  url: https://10.61.250.74
+  insecure: True
+  library: eks-a-templates
+  template: eks-a-admin-test
+  datacenter: Datacenter
+  datastore: datastore2
+  resourcePool: TestResourcePool
+  network: VM Network
+  folder: gwesterf

cmd/integration_test/cmd/run.go

@@ -14,11 +14,9 @@ import (
 )
 
 const (
-	amiIdFlagName              = "ami-id"
 	storageBucketFlagName      = "storage-bucket"
 	jobIdFlagName              = "job-id"
 	instanceProfileFlagName    = "instance-profile-name"
-	subnetIdFlagName           = "subnet-id"
 	regexFlagName              = "regex"
 	maxInstancesFlagName       = "max-instances"
 	maxConcurrentTestsFlagName = "max-concurrent-tests"
@@ -27,6 +25,7 @@ const (
 	cleanupVmsFlagName         = "cleanup-vms"
 	testReportFolderFlagName   = "test-report-folder"
 	branchNameFlagName         = "branch-name"
+	instanceConfigFlagName     = "instance-config"
 )
 
 var runE2ECmd = &cobra.Command{
@@ -44,7 +43,7 @@ var runE2ECmd = &cobra.Command{
 	},
 }
 
-var requiredFlags = []string{amiIdFlagName, storageBucketFlagName, jobIdFlagName, instanceProfileFlagName}
+var requiredFlags = []string{instanceConfigFlagName, storageBucketFlagName, jobIdFlagName, instanceProfileFlagName}
 
 func preRunSetup(cmd *cobra.Command, args []string) {
 	cmd.Flags().VisitAll(func(flag *pflag.Flag) {
@@ -57,18 +56,17 @@ func preRunSetup(cmd *cobra.Command, args []string) {
 
 func init() {
 	integrationTestCmd.AddCommand(runE2ECmd)
-	runE2ECmd.Flags().StringP(amiIdFlagName, "a", "", "Ami id")
+	runE2ECmd.Flags().StringP(instanceConfigFlagName, "c", "", "File path to the instance-config.yml config")
 	runE2ECmd.Flags().StringP(storageBucketFlagName, "s", "", "S3 bucket name to store eks-a binary")
 	runE2ECmd.Flags().StringP(jobIdFlagName, "j", "", "Id of the job being run")
 	runE2ECmd.Flags().StringP(instanceProfileFlagName, "i", "", "IAM instance profile name to attach to ec2 instances")
-	runE2ECmd.Flags().StringP(subnetIdFlagName, "n", "", "EC2 subnet ID")
 	runE2ECmd.Flags().StringP(regexFlagName, "r", "", "Run only those tests and examples matching the regular expression. Equivalent to go test -run")
 	runE2ECmd.Flags().IntP(maxInstancesFlagName, "m", 1, "Run tests in parallel on same instance within the max EC2 instance count")
-	runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "c", 1, "Maximum number of parallel tests that can be run at a time")
+	runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "p", 1, "Maximum number of parallel tests that can be run at a time")
 	runE2ECmd.Flags().StringSlice(skipFlagName, nil, "List of tests to skip")
 	runE2ECmd.Flags().Bool(bundlesOverrideFlagName, false, "Flag to indicate if the tests should run with a bundles override")
 	runE2ECmd.Flags().Bool(cleanupVmsFlagName, false, "Flag to indicate if VSphere VMs should be cleaned up automatically as tests complete")
-	runE2ECmd.Flags().String(testReportFolderFlagName, "", "Folder destination fo JUnit tests reports")
+	runE2ECmd.Flags().String(testReportFolderFlagName, "", "Folder destination for JUnit tests reports")
 	runE2ECmd.Flags().String(branchNameFlagName, "main", "EKS-A origin branch from where the tests are being run")
 
 	for _, flag := range requiredFlags {
@@ -79,11 +77,10 @@ func init() {
 }
 
 func runE2E(ctx context.Context) error {
-	amiId := viper.GetString(amiIdFlagName)
+	instanceConfigFile := viper.GetString(instanceConfigFlagName)
 	storageBucket := viper.GetString(storageBucketFlagName)
 	jobId := viper.GetString(jobIdFlagName)
 	instanceProfileName := viper.GetString(instanceProfileFlagName)
-	subnetId := viper.GetString(subnetIdFlagName)
 	testRegex := viper.GetString(regexFlagName)
 	maxInstances := viper.GetInt(maxInstancesFlagName)
 	maxConcurrentTests := viper.GetInt(maxConcurrentTestsFlagName)
@@ -94,19 +91,18 @@ func runE2E(ctx context.Context) error {
 	branchName := viper.GetString(branchNameFlagName)
 
 	runConf := e2e.ParallelRunConf{
-		MaxInstances:        maxInstances,
-		MaxConcurrentTests:  maxConcurrentTests,
-		AmiId:               amiId,
-		InstanceProfileName: instanceProfileName,
-		StorageBucket:       storageBucket,
-		JobId:               jobId,
-		SubnetId:            subnetId,
-		Regex:               testRegex,
-		TestsToSkip:         testsToSkip,
-		BundlesOverride:     bundlesOverride,
-		CleanupVms:          cleanupVms,
-		TestReportFolder:    testReportFolder,
-		BranchName:          branchName,
+		MaxInstances:           maxInstances,
+		MaxConcurrentTests:     maxConcurrentTests,
+		InstanceProfileName:    instanceProfileName,
+		StorageBucket:          storageBucket,
+		JobId:                  jobId,
+		Regex:                  testRegex,
+		TestsToSkip:            testsToSkip,
+		BundlesOverride:        bundlesOverride,
+		CleanupVms:             cleanupVms,
+		TestReportFolder:       testReportFolder,
+		BranchName:             branchName,
+		TestInstanceConfigFile: instanceConfigFile,
 	}
 
 	err := e2e.RunTestsInParallel(runConf)

internal/test/e2e/run.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/eks-anywhere/internal/pkg/ec2"
 	"github.com/aws/eks-anywhere/internal/pkg/ssm"
 	"github.com/aws/eks-anywhere/pkg/logger"
@@ -23,19 +24,18 @@ const (
 )
 
 type ParallelRunConf struct {
-	MaxInstances        int
-	MaxConcurrentTests  int
-	AmiId               string
-	InstanceProfileName string
-	StorageBucket       string
-	JobId               string
-	SubnetId            string
-	Regex               string
-	TestsToSkip         []string
-	BundlesOverride     bool
-	CleanupVms          bool
-	TestReportFolder    string
-	BranchName          string
+	TestInstanceConfigFile string
+	MaxInstances           int
+	MaxConcurrentTests     int
+	InstanceProfileName    string
+	StorageBucket          string
+	JobId                  string
+	Regex                  string
+	TestsToSkip            []string
+	BundlesOverride        bool
+	CleanupVms             bool
+	TestReportFolder       string
+	BranchName             string
 }
 
 type (
@@ -63,7 +63,11 @@ func RunTestsInParallel(conf ParallelRunConf) error {
 
 	var wg sync.WaitGroup
 
-	instancesConf := splitTests(testsList, conf)
+	instancesConf, err := splitTests(testsList, conf)
+	if err != nil {
+		return fmt.Errorf("failed to split tests: %v", err)
+	}
+
 	results := make([]instanceTestsResults, 0, len(instancesConf))
 	logTestGroups(instancesConf)
 	maxConcurrentTests := conf.MaxConcurrentTests
@@ -119,14 +123,23 @@ func RunTestsInParallel(conf ParallelRunConf) error {
 }
 
 type instanceRunConf struct {
-	amiId, instanceProfileName, storageBucket, jobId, parentJobId, subnetId, regex, instanceId string
-	testReportFolder, branchName                                                               string
-	ipPool                                                                                     networkutils.IPPool
-	bundlesOverride                                                                            bool
+	session                                                                   *session.Session
+	instanceProfileName, storageBucket, jobId, parentJobId, regex, instanceId string
+	testReportFolder, branchName                                              string
+	ipPool                                                                    networkutils.IPPool
+	bundlesOverride                                                           bool
+	testRunnerType                                                            TestRunnerType
+	testRunnerConfig                                                          TestInfraConfig
 }
 
 func RunTests(conf instanceRunConf) (testInstanceID string, testCommandResult *testCommandResult, err error) {
-	session, err := newSessionFromConf(conf)
+	testRunner := newTestRunner(conf.testRunnerType, conf.testRunnerConfig)
+	instanceId, err := testRunner.createInstance(conf)
+	if err != nil {
+		return "", nil, err
+	}
+
+	session, err := newE2ESession(instanceId, conf)
 	if err != nil {
 		return "", nil, err
 	}
@@ -207,13 +220,14 @@ func (e *E2ESession) commandWithEnvVars(command string) string {
 	return strings.Join(fullCommand, "; ")
 }
 
-func splitTests(testsList []string, conf ParallelRunConf) []instanceRunConf {
+func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, error) {
 	testPerInstance := len(testsList) / conf.MaxInstances
 	if testPerInstance == 0 {
 		testPerInstance = 1
 	}
 
 	vsphereTestsRe := regexp.MustCompile(vsphereRegex)
+	tinkerbellTestsRe := regexp.MustCompile(tinkerbellTestsRe)
 	privateNetworkTestsRe := regexp.MustCompile(`^.*(Proxy|RegistryMirror).*$`)
 	multiClusterTestsRe := regexp.MustCompile(`^.*Multicluster.*$`)
 
@@ -239,26 +253,42 @@ func splitTests(testsList []string, conf ParallelRunConf) []instanceRunConf {
 			}
 		}
 
+		awsSession, err := session.NewSession()
+		if err != nil {
+			return nil, fmt.Errorf("creating aws session for test: %s, %v", testName, err)
+		}
+
+		testRunnerConfig, err := NewTestRunnerConfigFromFile(conf.TestInstanceConfigFile)
+		if err != nil {
+			return nil, fmt.Errorf("creating test runner config for test: %s, %v", testName, err)
+		}
+
+		testRunnerType := Ec2TestRunnerType
+		if tinkerbellTestsRe.MatchString(testName) {
+			testRunnerType = VSphereTestRunnerType
+		}
+
 		if len(testsInCurrentInstance) == testPerInstance || (len(testsList)-1) == i {
 			runConfs = append(runConfs, instanceRunConf{
-				amiId:               conf.AmiId,
+				session:             awsSession,
 				instanceProfileName: conf.InstanceProfileName,
 				storageBucket:       conf.StorageBucket,
 				jobId:               fmt.Sprintf("%s-%d", conf.JobId, len(runConfs)),
 				parentJobId:         conf.JobId,
-				subnetId:            conf.SubnetId,
 				regex:               strings.Join(testsInCurrentInstance, "|"),
 				bundlesOverride:     conf.BundlesOverride,
 				testReportFolder:    conf.TestReportFolder,
 				branchName:          conf.BranchName,
 				ipPool:              ips,
+				testRunnerType:      testRunnerType,
+				testRunnerConfig:    *testRunnerConfig,
 			})
 
 			testsInCurrentInstance = make([]string, 0, testPerInstance)
 		}
 	}
 
-	return runConfs
+	return runConfs, nil
 }
 
 func logTestGroups(instancesConf []instanceRunConf) {

internal/test/e2e/setup.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws/session"
 
-	"github.com/aws/eks-anywhere/internal/pkg/ec2"
 	"github.com/aws/eks-anywhere/internal/pkg/s3"
 	"github.com/aws/eks-anywhere/internal/pkg/ssm"
 	"github.com/aws/eks-anywhere/pkg/logger"
@@ -25,15 +24,15 @@ const (
 	bundlesReleaseManifestFile = "local-bundle-release.yaml"
 	eksAComponentsManifestFile = "local-eksa-components.yaml"
 	testNameFile               = "e2e-test-name"
+	key                        = "Integration-Test"
+	tag                        = "EKSA-E2E"
 )
 
 type E2ESession struct {
 	session             *session.Session
-	amiId               string
 	instanceProfileName string
 	storageBucket       string
 	jobId               string
-	subnetId            string
 	instanceId          string
 	ipPool              networkutils.IPPool
 	testEnvVars         map[string]string
@@ -42,19 +41,13 @@ type E2ESession struct {
 	branchName          string
 }
 
-func newSessionFromConf(conf instanceRunConf) (*E2ESession, error) {
-	session, err := session.NewSession()
-	if err != nil {
-		return nil, fmt.Errorf("creating session: %v", err)
-	}
-
+func newE2ESession(instanceId string, conf instanceRunConf) (*E2ESession, error) {
 	e := &E2ESession{
-		session:             session,
-		amiId:               conf.amiId,
+		session:             conf.session,
+		instanceId:          instanceId,
 		instanceProfileName: conf.instanceProfileName,
 		storageBucket:       conf.storageBucket,
 		jobId:               conf.jobId,
-		subnetId:            conf.subnetId,
 		ipPool:              conf.ipPool,
 		testEnvVars:         make(map[string]string),
 		bundlesOverride:     conf.bundlesOverride,
@@ -71,19 +64,8 @@ func (e *E2ESession) setup(regex string) error {
 		return err
 	}
 
-	key := "Integration-Test"
-	tag := "EKSA-E2E"
-	name := fmt.Sprintf("eksa-e2e-%s", e.jobId)
-	logger.V(1).Info("Creating ec2 instance", "name", name)
-	instanceId, err := ec2.CreateInstance(e.session, e.amiId, key, tag, e.instanceProfileName, e.subnetId, name)
-	if err != nil {
-		return fmt.Errorf("creating instance for e2e tests: %v", err)
-	}
-	logger.V(1).Info("Instance created", "instance-id", instanceId)
-	e.instanceId = instanceId
-
 	logger.V(1).Info("Waiting until SSM is ready")
-	err = ssm.WaitForSSMReady(e.session, instanceId)
+	err = ssm.WaitForSSMReady(e.session, e.instanceId)
 	if err != nil {
 		return fmt.Errorf("waiting for ssm in new instance: %v", err)
 	}
@@ -108,6 +90,11 @@ func (e *E2ESession) setup(regex string) error {
 		return err
 	}
 
+	err = e.setupTinkerbellEnv(regex)
+	if err != nil {
+		return err
+	}
+
 	err = e.setupCloudStackEnv(regex)
 	if err != nil {
 		return err