account for multiple tests in single instance

decomm test runner

internal/pkg/ssm/activation.go

@@ -28,3 +28,18 @@ func CreateActivation(session *session.Session, instanceName, role string) (*Act
 
 	return &ActivationInfo{ActivationCode: *result.ActivationCode, ActivationID: *result.ActivationId}, nil
 }
+
+func DeleteActivation(session *session.Session, activationId string) (*ssm.DeleteActivationOutput, error) {
+	s := ssm.New(session)
+
+	request := ssm.DeleteActivationInput{
+		ActivationId: &activationId,
+	}
+
+	result, err := s.DeleteActivation(&request)
+	if err != nil {
+		return nil, fmt.Errorf("failed to delete ssm activation: %v", err)
+	}
+
+	return result, nil
+}

internal/pkg/ssm/instance.go

@@ -29,3 +29,17 @@ func GetInstanceByActivationId(session *session.Session, id string) (*ssm.Instan
 
 	return infoList[0], nil
 }
+
+func DeregisterInstance(session *session.Session, id string) (*ssm.DeregisterManagedInstanceOutput, error) {
+	s := ssm.New(session)
+	input := ssm.DeregisterManagedInstanceInput{
+		InstanceId: &id,
+	}
+
+	output, err := s.DeregisterManagedInstance(&input)
+	if err != nil {
+		return nil, fmt.Errorf("failed to deregister ssm instance %s: %v", id, err)
+	}
+
+	return output, nil
+}

internal/test/e2e/artifacts.go

@@ -14,7 +14,7 @@ const e2eHomeFolder = "/home/e2e/"
 func (e *E2ESession) uploadGeneratedFilesFromInstance(testName string) {
 	logger.V(1).Info("Uploading log files to s3 bucket")
 	command := newCopyCommand().from(
-		e2eHomeFolder, clusterName(e.branchName, e.instanceId),
+		e2eHomeFolder, clusterName(e.branchName, e.instanceId, testName),
 	).to(
 		e.generatedArtifactsBucketPath(), testName,
 	).recursive().String()
@@ -29,7 +29,7 @@ func (e *E2ESession) uploadGeneratedFilesFromInstance(testName string) {
 func (e *E2ESession) uploadDiagnosticArchiveFromInstance(testName string) {
 	bundleNameFormat := "support-bundle-*.tar.gz"
 	logger.V(1).Info("Uploading diagnostic bundle to s3 bucket")
-	command := newCopyCommand().from(e2eHomeFolder).to(
+	command := newCopyCommand().from(e2eHomeFolder, clusterName(e.branchName, e.instanceId, testName)).to(
 		e.generatedArtifactsBucketPath(), testName,
 	).recursive().exclude("*").include(bundleNameFormat).String()
 

internal/test/e2e/run.go

@@ -108,11 +108,15 @@ func RunTestsInParallel(conf ParallelRunConf) error {
 			logger.Info("Instance tests completed successfully", "jobId", r.conf.jobId, "instanceId", r.conf.instanceId, "commandId", r.testCommandResult.CommandId, "tests", r.conf.regex, "status", passedStatus)
 			logResult(r.testCommandResult)
 		}
-		clusterName := clusterName(r.conf.branchName, r.conf.instanceId)
-		if conf.CleanupVms {
-			err = CleanUpVsphereTestResources(context.Background(), clusterName)
-			if err != nil {
-				logger.Error(err, "Failed to clean up VSphere cluster VMs", "clusterName", r.conf.instanceId)
+		regex := strings.Trim(r.conf.regex, "\"")
+		testCases := strings.Split(regex, "|")
+		for _, testName := range testCases {
+			clusterName := clusterName(r.conf.branchName, r.conf.instanceId, testName)
+			if conf.CleanupVms {
+				err = CleanUpVsphereTestResources(context.Background(), clusterName)
+				if err != nil {
+					logger.Error(err, "Failed to clean up VSphere cluster VMs", "clusterName", r.conf.instanceId)
+				}
 			}
 		}
 	}
@@ -142,6 +146,13 @@ func RunTests(conf instanceRunConf) (testInstanceID string, testCommandResult *t
 		return "", nil, err
 	}
 
+	defer func() {
+		err := testRunner.decommInstance(conf)
+		if err != nil {
+			logger.V(1).Info("WARN: Failed to decomm e2e test runner instance", "error", err)
+		}
+	}()
+
 	session, err := newE2ESession(instanceId, conf)
 	if err != nil {
 		return "", nil, err
@@ -176,7 +187,7 @@ func (e *E2ESession) runTests(regex string) (testCommandResult *testCommandResul
 	command := "GOVERSION=go1.16.6 gotestsum --junitfile=junit-testing.xml --raw-command --format=standard-verbose --hide-summary=all --ignore-non-json-output-lines -- test2json -t -p e2e ./bin/e2e.test -test.v"
 
 	if regex != "" {
-		command = fmt.Sprintf("%s -test.run %s", command, regex)
+		command = fmt.Sprintf("%s -test.run \"%s\"", command, regex)
 	}
 
 	command = e.commandWithEnvVars(command)
@@ -197,16 +208,20 @@ func (e *E2ESession) runTests(regex string) (testCommandResult *testCommandResul
 }
 
 func (c instanceRunConf) runPostTestsProcessing(e *E2ESession, testCommandResult *testCommandResult) error {
-	testName := strings.Trim(c.regex, "\"")
-	e.uploadJUnitReportFromInstance(testName)
-	if c.testReportFolder != "" {
-		e.downloadJUnitReportToLocalDisk(testName, c.testReportFolder)
-	}
+	regex := strings.Trim(c.regex, "\"")
+	tests := strings.Split(regex, "|")
 
-	if !testCommandResult.Successful() {
-		e.uploadGeneratedFilesFromInstance(testName)
-		e.uploadDiagnosticArchiveFromInstance(testName)
-		return nil
+	for _, testName := range tests {
+		e.uploadJUnitReportFromInstance(testName)
+		if c.testReportFolder != "" {
+			e.downloadJUnitReportToLocalDisk(testName, c.testReportFolder)
+		}
+
+		if !testCommandResult.Successful() {
+			e.uploadGeneratedFilesFromInstance(testName)
+			e.uploadDiagnosticArchiveFromInstance(testName)
+			return nil
+		}
 	}
 
 	return nil

internal/test/e2e/setup.go

@@ -136,7 +136,7 @@ func (e *E2ESession) setup(regex string) error {
 		e.testEnvVars[e2etests.BranchNameEnvVar] = e.branchName
 	}
 
-	e.testEnvVars[e2etests.ClusterNameVar] = clusterName(e.branchName, e.instanceId)
+	e.testEnvVars[e2etests.ClusterPrefixVar] = clusterPrefix(e.branchName, e.instanceId)
 	return nil
 }
 
@@ -207,17 +207,21 @@ func (e *E2ESession) createTestNameFile(testName string) error {
 	return nil
 }
 
-func clusterName(branch string, instanceId string) (clusterName string) {
+func clusterPrefix(branch, instanceId string) (clusterPrefix string) {
 	if branch == "" {
 		return instanceId
 	}
-	clusterNameTemplate := "%s-%s"
 	forbiddenChars := []string{"."}
 	sanitizedBranch := strings.ToLower(branch)
 	for _, char := range forbiddenChars {
 		sanitizedBranch = strings.ReplaceAll(sanitizedBranch, char, "-")
 	}
-	clusterName = fmt.Sprintf(clusterNameTemplate, sanitizedBranch, instanceId)
+	clusterPrefix = fmt.Sprintf("%s-%s", sanitizedBranch, instanceId)
+	return clusterPrefix
+}
+
+func clusterName(branch, instanceId, testName string) (clusterName string) {
+	clusterName = fmt.Sprintf("%s-%s", clusterPrefix(branch, instanceId), e2etests.GetTestNameHash(testName))
 	if len(clusterName) > 80 {
 		logger.Info("Cluster name is longer than 80 characters; truncating to 80 characters.", "original cluster name", clusterName, "truncated cluster name", clusterName[:80])
 		clusterName = clusterName[:80]

internal/test/e2e/testRunner.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"strconv"
@@ -30,6 +31,7 @@ const (
 type TestRunner interface {
 	createInstance(instanceConf instanceRunConf) (string, error)
 	tagInstance(instanceConf instanceRunConf, key, value string) error
+	decommInstance(instanceRunConf) error
 }
 
 type TestRunnerType string
@@ -41,9 +43,9 @@ const (
 
 func newTestRunner(runnerType TestRunnerType, config TestInfraConfig) TestRunner {
 	if runnerType == VSphereTestRunnerType {
-		return config.VSphereTestRunner
+		return &config.VSphereTestRunner
 	} else {
-		return config.Ec2TestRunner
+		return &config.Ec2TestRunner
 	}
 }
 
@@ -80,6 +82,7 @@ type Ec2TestRunner struct {
 
 type VSphereTestRunner struct {
 	testRunner
+	ActivationId string
 	Url          string `yaml:"url"`
 	Insecure     bool   `yaml:"insecure"`
 	Library      string `yaml:"library"`
@@ -119,7 +122,7 @@ func (v *VSphereTestRunner) setEnvironment() error {
 	return nil
 }
 
-func (v VSphereTestRunner) createInstance(c instanceRunConf) (string, error) {
+func (v *VSphereTestRunner) createInstance(c instanceRunConf) (string, error) {
 	name := getTestRunnerName(c.jobId)
 	logger.V(1).Info("Creating vSphere Test Runner instance", "name", name)
 
@@ -165,11 +168,14 @@ func (v VSphereTestRunner) createInstance(c instanceRunConf) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("waiting for ssm instance to activate %s : %v", name, err)
 	}
+
 	v.InstanceID = *ssmInstance.InstanceId
+	v.ActivationId = ssmActivationInfo.ActivationID
+
 	return *ssmInstance.InstanceId, nil
 }
 
-func (e Ec2TestRunner) createInstance(c instanceRunConf) (string, error) {
+func (e *Ec2TestRunner) createInstance(c instanceRunConf) (string, error) {
 	name := getTestRunnerName(c.jobId)
 	logger.V(1).Info("Creating ec2 Test Runner instance", "name", name)
 	instanceId, err := ec2.CreateInstance(c.session, e.AmiID, key, tag, c.instanceProfileName, e.SubnetID, name)
@@ -181,7 +187,7 @@ func (e Ec2TestRunner) createInstance(c instanceRunConf) (string, error) {
 	return instanceId, nil
 }
 
-func (v VSphereTestRunner) tagInstance(c instanceRunConf, key, value string) error {
+func (v *VSphereTestRunner) tagInstance(c instanceRunConf, key, value string) error {
 	vmName := getTestRunnerName(c.jobId)
 	vmPath := fmt.Sprintf("/%s/vm/%s/%s", v.Datacenter, v.Folder, vmName)
 	tag := fmt.Sprintf("%s:%s", key, value)
@@ -191,14 +197,38 @@ func (v VSphereTestRunner) tagInstance(c instanceRunConf, key, value string) err
 	return nil
 }
 
-func (e Ec2TestRunner) tagInstance(c instanceRunConf, key, value string) error {
+func (e *Ec2TestRunner) tagInstance(c instanceRunConf, key, value string) error {
 	err := ec2.TagInstance(c.session, c.instanceId, key, value)
 	if err != nil {
 		return fmt.Errorf("failed to tag Ec2 test runner: %v", err)
 	}
 	return nil
 }
 
+func (v *VSphereTestRunner) decommInstance(c instanceRunConf) error {
+	_, deregisterError := ssm.DeregisterInstance(c.session, v.InstanceID)
+	_, deactivateError := ssm.DeleteActivation(c.session, v.ActivationId)
+	deleteError := vsphereRmVms(context.Background(), getTestRunnerName(c.jobId))
+
+	if deregisterError != nil {
+		return fmt.Errorf("failed to decommission vsphere test runner ssm instance: %v", deregisterError)
+	}
+
+	if deactivateError != nil {
+		return fmt.Errorf("failed to decommission vsphere test runner ssm instance: %v", deactivateError)
+	}
+
+	if deleteError != nil {
+		return fmt.Errorf("failed to decommission vsphere test runner ssm instance: %v", deleteError)
+	}
+
+	return nil
+}
+
+func (v *Ec2TestRunner) decommInstance(c instanceRunConf) error {
+	return nil
+}
+
 func getTestRunnerName(jobId string) string {
 	return fmt.Sprintf("eksa-e2e-%s", jobId)
 }

test/framework/cluster.go

@@ -35,7 +35,7 @@ const (
 	defaultClusterName               = "eksa-test"
 	eksctlVersionEnvVar              = "EKSCTL_VERSION"
 	eksctlVersionEnvVarDummyVal      = "ham sandwich"
-	ClusterNameVar                   = "T_CLUSTER_NAME"
+	ClusterPrefixVar                 = "T_CLUSTER_PREFIX"
 	JobIdVar                         = "T_JOB_ID"
 	BundlesOverrideVar               = "T_BUNDLES_OVERRIDE"
 	hardwareYamlPath                 = "hardware.yaml"
@@ -85,7 +85,7 @@ func NewClusterE2ETest(t *testing.T, provider Provider, opts ...ClusterE2ETestOp
 		eksaBinaryLocation:    defaultEksaBinaryLocation,
 	}
 
-	e.ClusterConfigFolder = fmt.Sprintf("%s-config", e.ClusterName)
+	e.ClusterConfigFolder = e.ClusterName
 	e.HardwareConfigLocation = filepath.Join(e.ClusterConfigFolder, hardwareYamlPath)
 	e.HardwareCsvLocation = filepath.Join(e.ClusterConfigFolder, hardwareCsvPath)
 
@@ -581,17 +581,22 @@ func (e *ClusterE2ETest) getJobIdFromEnv() string {
 	return os.Getenv(JobIdVar)
 }
 
+func GetTestNameHash(name string) string {
+	h := sha1.New()
+	h.Write([]byte(name))
+	testNameHash := fmt.Sprintf("%x", h.Sum(nil))
+	return testNameHash[:7]
+}
+
 func getClusterName(t *testing.T) string {
-	value := os.Getenv(ClusterNameVar)
+	value := os.Getenv(ClusterPrefixVar)
+	// Append hash to make each cluster name unique per test. Using the testname will be too long
+	// and would fail validations
 	if len(value) == 0 {
-		h := sha1.New()
-		h.Write([]byte(t.Name()))
-		testNameHash := fmt.Sprintf("%x", h.Sum(nil))
-		// Append hash to make each cluster name unique per test. Using the testname will be too long
-		// and would fail validations
-		return fmt.Sprintf("%s-%s", defaultClusterName, testNameHash[:7])
-	}
-	return value
+		value = defaultClusterName
+	}
+
+	return fmt.Sprintf("%s-%s", value, GetTestNameHash(t.Name()))
 }
 
 func getBundlesOverride() string {