add tinkerbell ci test harness to e2e test framework (aws#2031)

* add tinkerbell ci test harness to e2e test framework

* split tink tests into instance suites based on hardware capacity

* account for multiple tests in single instance

decomm test runner

* fix generate hardware & marshall labels

* provision tink boostrap and server ip for ci

* update codebuild job for tinkerbell ci

* fix linter

* add kube 1.23 test

* account for future tinkerbell boostrap ip arg during create cluster cmd

* remove tink feature flag enable

* add option to override the local tinkerbell IP in the bootstrap cluster

* update tests for bootstrap flag

* update to lab vcenter

* update max cluster name length and power off hardware prior to each test

* add br test for tink

* enable k8 1.23 test for tink

* update conformance and prow tests to accound for tinkerbell changes

* split tink and cp cidr ranges

* persist vm on failure for debug

* fix bundle upload

* update tink conformance tests

* remove example infra config

* skip three worker test

* add ping test to show connectivity to vcenter in colo lab thru direct connect

* remove ping

* only run tink tests on main branch

* update test concurrent count

Makefile

@@ -118,6 +118,7 @@ ORGANIZE_BINARIES_TARGETS = $(addsuffix /eks-a-tools,$(addprefix $(BINARY_DEPS_D
 
 EKS_A_PLATFORMS ?= linux-amd64 linux-arm64 darwin-arm64 darwin-amd64
 EKS_A_CROSS_PLATFORMS := $(foreach platform,$(EKS_A_PLATFORMS),eks-a-cross-platform-$(platform))
+E2E_CROSS_PLATFORMS := $(foreach platform,$(EKS_A_PLATFORMS),e2e-cross-platform-$(platform))
 EKS_A_RELEASE_CROSS_PLATFORMS := $(foreach platform,$(EKS_A_PLATFORMS),eks-a-release-cross-platform-$(platform))
 
 DOCKER_E2E_TEST := TestDockerKubernetes121SimpleFlow
@@ -465,19 +466,20 @@ verify-mocks: mocks ## Verify if mocks need to be updated
 		exit 1;\
 	fi
 
+.PHONY: e2e-cross-platform
+e2e-cross-platform: $(E2E_CROSS_PLATFORMS)
+
+.PHONY: e2e-cross-platform-%
+e2e-cross-platform-%: ## Generate binaries for Linux and MacOS
+e2e-cross-platform-%: GO_OS = $(firstword $(subst -, ,$*))
+e2e-cross-platform-%: GO_ARCH = $(lastword $(subst -, ,$*))
+e2e-cross-platform-%:
+	$(MAKE) e2e-tests-binary E2E_TAGS=e2e GIT_VERSION=$(DEV_GIT_VERSION) GO_OS=$(GO_OS) GO_ARCH=$(GO_ARCH) E2E_OUTPUT_FILE=bin/$(GO_OS)/$(GO_ARCH)/e2e.test
+
 .PHONY: e2e
 e2e: eks-a-e2e integration-test-binary ## Build integration tests
 	$(MAKE) e2e-tests-binary E2E_TAGS=e2e
 
-.PHONY: conformance
-conformance:
-	$(MAKE) e2e-tests-binary E2E_TAGS=conformance_e2e
-	./bin/e2e.test -test.v -test.run 'TestVSphereKubernetes121ThreeWorkersConformanc.*'
-
-.PHONY: conformance-tests
-conformance-tests: eks-a-e2e integration-test-binary ## Build e2e conformance tests
-	$(MAKE) e2e-tests-binary E2E_TAGS=conformance_e2e
-
 .PHONY: eks-a-e2e
 eks-a-e2e:
 	if [ "$(CODEBUILD_CI)" = "true" ]; then \
@@ -496,12 +498,22 @@ eks-a-e2e:
 	fi
 
 .PHONY: e2e-tests-binary
+e2e-tests-binary: E2E_OUTPUT_FILE ?= bin/e2e.test
 e2e-tests-binary:
-	$(GO) test ./test/e2e -c -o bin/e2e.test -tags "$(E2E_TAGS)" -ldflags "-X github.com/aws/eks-anywhere/pkg/version.gitVersion=$(DEV_GIT_VERSION) -X github.com/aws/eks-anywhere/pkg/cluster.releasesManifestURL=$(RELEASE_MANIFEST_URL) -X github.com/aws/eks-anywhere/pkg/manifests/releases.manifestURL=$(RELEASE_MANIFEST_URL)"
+	GOOS=$(GO_OS) GOARCH=$(GO_ARCH) $(GO) test ./test/e2e -c -o bin/e2e.test -tags "$(E2E_TAGS)" -ldflags "-X github.com/aws/eks-anywhere/pkg/version.gitVersion=$(DEV_GIT_VERSION) -X github.com/aws/eks-anywhere/pkg/cluster.releasesManifestURL=$(RELEASE_MANIFEST_URL) -X github.com/aws/eks-anywhere/pkg/manifests/releases.manifestURL=$(RELEASE_MANIFEST_URL)"
 
 .PHONY: integration-test-binary
 integration-test-binary:
-	$(GO) build -o bin/test github.com/aws/eks-anywhere/cmd/integration_test
+	GOOS=$(GO_OS) GOARCH=$(GO_ARCH) $(GO) build -o bin/test github.com/aws/eks-anywhere/cmd/integration_test
+
+.PHONY: conformance
+conformance:
+	$(MAKE) e2e-tests-binary E2E_TAGS=conformance_e2e
+	./bin/e2e.test -test.v -test.run 'TestVSphereKubernetes121ThreeWorkersConformanc.*'
+
+.PHONY: conformance-tests
+conformance-tests: eks-a-e2e integration-test-binary ## Build e2e conformance tests
+	$(MAKE) e2e-tests-binary E2E_TAGS=conformance_e2e
 
 .PHONY: check-eksa-components-override
 check-eksa-components-override:

cmd/eksctl-anywhere/cmd/generatehardware.go

@@ -32,7 +32,7 @@ func init() {
 	flags := generateHardwareCmd.Flags()
 	flags.StringVarP(&hOpts.outputPath, "output", "o", "", "Path to output hardware YAML.")
 	flags.StringVarP(
-		&cc.hardwareCSVPath,
+		&hOpts.csvPath,
 		TinkerbellHardwareCSVFlagName,
 		TinkerbellHardwareCSVFlagAlias,
 		"",

cmd/integration_test/build/buildspecs/conformance-eks-a-cli.yml

@@ -4,6 +4,12 @@ env:
   variables:
     INTEGRATION_TEST_MAX_EC2_COUNT: 25
     INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 25
+    T_TINKERBELL_MAX_HARDWARE_PER_TEST: 5
+    T_TINKERBELL_INVENTORY_CSV: "hardware-manifests/inventory.csv"
+    T_TINKERBELL_BOOTSTRAP_INTERFACE: "ens192"
+    TEST_RUNNER_GOVC_LIBRARY: "eks-a-templates"
+    TEST_RUNNER_GOVC_TEMPLATE: "eks-a-admin-ci"
+    INTEGRATION_TEST_INFRA_CONFIG: "/tmp/test-infra.yml"
   secrets-manager:
     EKSA_VSPHERE_USERNAME: "vsphere_ci_beta_connection:vsphere_username"
     EKSA_VSPHERE_PASSWORD: "vsphere_ci_beta_connection:vsphere_password"
@@ -33,24 +39,40 @@ env:
     T_VSPHERE_TEMPLATE_BR_1_23: "vsphere_ci_beta_connection:template_br_23"
     T_VSPHERE_TLS_INSECURE: "vsphere_ci_beta_connection:tls_insecure"
     T_VSPHERE_TLS_THUMBPRINT: "vsphere_ci_beta_connection:tls_thumbprint"
+    T_TINKERBELL_IMAGE_UBUNTU_1_20: "tinkerbell_ci:image_ubuntu_1_20"
+    T_TINKERBELL_IMAGE_UBUNTU_1_21: "tinkerbell_ci:image_ubuntu_1_21"
+    T_TINKERBELL_IMAGE_UBUNTU_1_22: "tinkerbell_ci:image_ubuntu_1_22"
+    T_TINKERBELL_IMAGE_UBUNTU_1_23: "tinkerbell_ci:image_ubuntu_1_23"
+    T_TINKERBELL_SSH_AUTHORIZED_KEY: "vsphere_ci_beta_connection:ssh_authorized_key"
+    T_TINKERBELL_NETWORK_CIDR: "tinkerbell_ci:network_cidr"
+    T_TINKERBELL_CP_NETWORK_CIDR: "tinkerbell_ci:cp_network_cidr"
+    T_TINKERBELL_S3_INVENTORY_CSV_KEY: "tinkerbell_ci:s3_inventory_csv"
+    TEST_RUNNER_GOVC_USERNAME: "tinkerbell_ci:govc_username"
+    TEST_RUNNER_GOVC_PASSWORD: "tinkerbell_ci: govc_password"
+    TEST_RUNNER_GOVC_URL: "tinkerbell_ci: govc_url"
+    TEST_RUNNER_GOVC_DATACENTER: "tinkerbell_ci: govc_datacenter"
+    TEST_RUNNER_GOVC_DATASTORE: "tinkerbell_ci: govc_datastore"
+    TEST_RUNNER_GOVC_RESOURCE_POOL: "tinkerbell_ci: govc_resource_pool"
+    TEST_RUNNER_GOVC_NETWORK: "tinkerbell_ci: govc_network"
+    TEST_RUNNER_GOVC_FOLDER: "tinkerbell_ci: govc_folder"
 
 phases:
   pre_build:
     commands:
       - source ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/setup_profile.sh
+      - source ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/create_infra_config.sh
       - ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/start_docker.sh
       - make conformance-tests
   build:
     commands:
       - export JOB_ID=$CODEBUILD_BUILD_ID
       - >
         ./bin/test e2e run
-        -a ${INTEGRATION_TEST_AL2_AMI_ID}
+        -c ${INTEGRATION_TEST_INFRA_CONFIG}
         -s ${INTEGRATION_TEST_STORAGE_BUCKET}
         -j ${JOB_ID}
         -i ${INTEGRATION_TEST_INSTANCE_PROFILE}
-        -n ${INTEGRATION_TEST_SUBNET_ID}
         -m ${INTEGRATION_TEST_MAX_EC2_COUNT}
-        -c ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
+        -p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
         -r 'Test'
         --cleanup-vms=true

cmd/integration_test/build/buildspecs/test-eks-a-cli.yml

@@ -2,13 +2,19 @@ version: 0.2
 
 env:
   variables:
-    INTEGRATION_TEST_MAX_EC2_COUNT: 130
-    INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 130
+    INTEGRATION_TEST_MAX_EC2_COUNT: 160
+    INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT: 160
     T_CLOUDSTACK_CIDR: "10.11.255.0/24"
-    SKIPPED_TESTS: "TestVSphereKubernetes122BottlerocketAutoimport,TestTinkerbellKubernetes121ForceFlow,TestTinkerbellKubernetes121ThreeReplicasTwoWorkersForceFlow,TestTinkerbellKubernetes122ForceFlow,TestTinkerbellKubernetes120SimpleFlow,TestTinkerbellKubernetes121SimpleFlow,TestTinkerbellKubernetes121ThreeReplicasTwoWorkersSimpleFlow,TestTinkerbellKubernetes121DellSimpleFlow,TestTinkerbellKubernetes121HPSimpleFlow,TestTinkerbellKubernetes121SuperMicroSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdThreeReplicasTwoWorkersSimpleFlow,TestCloudStackUpgradeMulticlusterWorkloadClusterWithFluxLegacy,TestCloudStackKubernetes121StackedEtcdRedhat,TestCloudStackKubernetes120RedhatWorkerNodeUpgrade,TestCloudStackKubernetes120RedhatTo121MultipleFieldsUpgrade,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC,TestSnowKubernetes121UbuntuProxyConfig"
+    SKIPPED_TESTS: "TestTinkerbellKubernetes121ThreeWorkersSimpleFlow,TestTinkerbellKubernetes122BottleRocketSimpleFlow,TestTinkerbellKubernetes121ExternalEtcdSimpleFlow,TestTinkerbellKubernetes123SimpleFlow,TestVSphereKubernetes122BottlerocketAutoimport,TestCloudStackUpgradeMulticlusterWorkloadClusterWithFluxLegacy,TestCloudStackKubernetes121StackedEtcdRedhat,TestCloudStackKubernetes120RedhatWorkerNodeUpgrade,TestCloudStackKubernetes120RedhatTo121MultipleFieldsUpgrade,TestCloudStackKubernetes121RedhatProxyConfig,TestCloudStackKubernetes120OIDC,TestCloudStackKubernetes121OIDC,TestCloudStackKubernetes121RedhatRegistryMirrorAndCert,TestSnowKubernetes121SimpleFlow,TestSnowKubernetes121OIDC,TestSnowKubernetes121UbuntuProxyConfig"
     CLOUDSTACK_PROVIDER: true
     EKSA_GIT_KNOWN_HOSTS: "/tmp/known_hosts"
     EKSA_GIT_PRIVATE_KEY: "/tmp/private-key"
+    T_TINKERBELL_MAX_HARDWARE_PER_TEST: 4
+    T_TINKERBELL_INVENTORY_CSV: "hardware-manifests/inventory.csv"
+    T_TINKERBELL_BOOTSTRAP_INTERFACE: "ens192"
+    TEST_RUNNER_GOVC_LIBRARY: "eks-a-templates"
+    TEST_RUNNER_GOVC_TEMPLATE: "eks-a-admin-ci"
+    INTEGRATION_TEST_INFRA_CONFIG: "/tmp/test-infra.yml"
   secrets-manager:
     EKSA_VSPHERE_USERNAME: "vsphere_ci_beta_connection:vsphere_username"
     EKSA_VSPHERE_PASSWORD: "vsphere_ci_beta_connection:vsphere_password"
@@ -65,11 +71,28 @@ env:
     T_CLOUDSTACK_POD_CIDR: "cloudstack_ci_beta_connection:pod_cidr"
     T_CLOUDSTACK_SERVICE_CIDR: "cloudstack_ci_beta_connection:service_cidr"
     T_CLOUDSTACK_SSH_AUTHORIZED_KEY: "vsphere_ci_beta_connection:ssh_authorized_key"
+    T_TINKERBELL_IMAGE_UBUNTU_1_20: "tinkerbell_ci:image_ubuntu_1_20"
+    T_TINKERBELL_IMAGE_UBUNTU_1_21: "tinkerbell_ci:image_ubuntu_1_21"
+    T_TINKERBELL_IMAGE_UBUNTU_1_22: "tinkerbell_ci:image_ubuntu_1_22"
+    T_TINKERBELL_IMAGE_UBUNTU_1_23: "tinkerbell_ci:image_ubuntu_1_23"
+    T_TINKERBELL_SSH_AUTHORIZED_KEY: "vsphere_ci_beta_connection:ssh_authorized_key"
+    T_TINKERBELL_NETWORK_CIDR: "tinkerbell_ci:network_cidr"
+    T_TINKERBELL_CP_NETWORK_CIDR: "tinkerbell_ci:cp_network_cidr"
+    T_TINKERBELL_S3_INVENTORY_CSV_KEY: "tinkerbell_ci:s3_inventory_csv"
+    TEST_RUNNER_GOVC_USERNAME: "tinkerbell_ci:govc_username"
+    TEST_RUNNER_GOVC_PASSWORD: "tinkerbell_ci: govc_password"
+    TEST_RUNNER_GOVC_URL: "tinkerbell_ci: govc_url"
+    TEST_RUNNER_GOVC_DATACENTER: "tinkerbell_ci: govc_datacenter"
+    TEST_RUNNER_GOVC_DATASTORE: "tinkerbell_ci: govc_datastore"
+    TEST_RUNNER_GOVC_RESOURCE_POOL: "tinkerbell_ci: govc_resource_pool"
+    TEST_RUNNER_GOVC_NETWORK: "tinkerbell_ci: govc_network"
+    TEST_RUNNER_GOVC_FOLDER: "tinkerbell_ci: govc_folder"
 
 phases:
   pre_build:
     commands:
       - source ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/setup_profile.sh
+      - source ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/create_infra_config.sh
       - ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/start_docker.sh
       - export CLUSTER_NAME_PREFIX="${BRANCH_NAME//./-}"
       - >
@@ -86,13 +109,12 @@ phases:
         fi
       - >
         ./bin/test e2e run
-        -a ${INTEGRATION_TEST_AL2_AMI_ID}
+        -c ${INTEGRATION_TEST_INFRA_CONFIG}
         -s ${INTEGRATION_TEST_STORAGE_BUCKET}
         -j ${JOB_ID}
         -i ${INTEGRATION_TEST_INSTANCE_PROFILE}
-        -n ${INTEGRATION_TEST_SUBNET_ID}
         -m ${INTEGRATION_TEST_MAX_EC2_COUNT}
-        -c ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
+        -p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
         -r 'Test'
         -v 4
         --skip ${SKIPPED_TESTS}

cmd/integration_test/build/script/create_infra_config.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -x
+set -o pipefail
+
+cat << EOF > ${INTEGRATION_TEST_INFRA_CONFIG}
+---
+
+ec2:
+  amiId: ${INTEGRATION_TEST_AL2_AMI_ID}
+  subnetId: ${INTEGRATION_TEST_SUBNET_ID}
+
+vSphere:
+  url: ${TEST_RUNNER_GOVC_URL}
+  insecure: True
+  library: ${TEST_RUNNER_GOVC_LIBRARY}
+  template: ${TEST_RUNNER_GOVC_TEMPLATE}
+  datacenter: ${TEST_RUNNER_GOVC_DATACENTER}
+  datastore: ${TEST_RUNNER_GOVC_DATASTORE}
+  resourcePool: ${TEST_RUNNER_GOVC_RESOURCE_POOL}
+  network: ${TEST_RUNNER_GOVC_NETWORK}
+  folder: ${TEST_RUNNER_GOVC_FOLDER}
+EOF

cmd/integration_test/cmd/run.go

@@ -14,11 +14,9 @@ import (
 )
 
 const (
-	amiIdFlagName              = "ami-id"
 	storageBucketFlagName      = "storage-bucket"
 	jobIdFlagName              = "job-id"
 	instanceProfileFlagName    = "instance-profile-name"
-	subnetIdFlagName           = "subnet-id"
 	regexFlagName              = "regex"
 	maxInstancesFlagName       = "max-instances"
 	maxConcurrentTestsFlagName = "max-concurrent-tests"
@@ -27,6 +25,7 @@ const (
 	cleanupVmsFlagName         = "cleanup-vms"
 	testReportFolderFlagName   = "test-report-folder"
 	branchNameFlagName         = "branch-name"
+	instanceConfigFlagName     = "instance-config"
 )
 
 var runE2ECmd = &cobra.Command{
@@ -44,7 +43,7 @@ var runE2ECmd = &cobra.Command{
 	},
 }
 
-var requiredFlags = []string{amiIdFlagName, storageBucketFlagName, jobIdFlagName, instanceProfileFlagName}
+var requiredFlags = []string{instanceConfigFlagName, storageBucketFlagName, jobIdFlagName, instanceProfileFlagName}
 
 func preRunSetup(cmd *cobra.Command, args []string) {
 	cmd.Flags().VisitAll(func(flag *pflag.Flag) {
@@ -57,18 +56,17 @@ func preRunSetup(cmd *cobra.Command, args []string) {
 
 func init() {
 	integrationTestCmd.AddCommand(runE2ECmd)
-	runE2ECmd.Flags().StringP(amiIdFlagName, "a", "", "Ami id")
+	runE2ECmd.Flags().StringP(instanceConfigFlagName, "c", "", "File path to the instance-config.yml config")
 	runE2ECmd.Flags().StringP(storageBucketFlagName, "s", "", "S3 bucket name to store eks-a binary")
 	runE2ECmd.Flags().StringP(jobIdFlagName, "j", "", "Id of the job being run")
-	runE2ECmd.Flags().StringP(instanceProfileFlagName, "i", "", "IAM instance profile name to attach to ec2 instances")
-	runE2ECmd.Flags().StringP(subnetIdFlagName, "n", "", "EC2 subnet ID")
+	runE2ECmd.Flags().StringP(instanceProfileFlagName, "i", "", "IAM instance profile name to attach to ssm instances")
 	runE2ECmd.Flags().StringP(regexFlagName, "r", "", "Run only those tests and examples matching the regular expression. Equivalent to go test -run")
 	runE2ECmd.Flags().IntP(maxInstancesFlagName, "m", 1, "Run tests in parallel on same instance within the max EC2 instance count")
-	runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "c", 1, "Maximum number of parallel tests that can be run at a time")
+	runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "p", 1, "Maximum number of parallel tests that can be run at a time")
 	runE2ECmd.Flags().StringSlice(skipFlagName, nil, "List of tests to skip")
 	runE2ECmd.Flags().Bool(bundlesOverrideFlagName, false, "Flag to indicate if the tests should run with a bundles override")
 	runE2ECmd.Flags().Bool(cleanupVmsFlagName, false, "Flag to indicate if VSphere VMs should be cleaned up automatically as tests complete")
-	runE2ECmd.Flags().String(testReportFolderFlagName, "", "Folder destination fo JUnit tests reports")
+	runE2ECmd.Flags().String(testReportFolderFlagName, "", "Folder destination for JUnit tests reports")
 	runE2ECmd.Flags().String(branchNameFlagName, "main", "EKS-A origin branch from where the tests are being run")
 
 	for _, flag := range requiredFlags {
@@ -79,11 +77,10 @@ func init() {
 }
 
 func runE2E(ctx context.Context) error {
-	amiId := viper.GetString(amiIdFlagName)
+	instanceConfigFile := viper.GetString(instanceConfigFlagName)
 	storageBucket := viper.GetString(storageBucketFlagName)
 	jobId := viper.GetString(jobIdFlagName)
 	instanceProfileName := viper.GetString(instanceProfileFlagName)
-	subnetId := viper.GetString(subnetIdFlagName)
 	testRegex := viper.GetString(regexFlagName)
 	maxInstances := viper.GetInt(maxInstancesFlagName)
 	maxConcurrentTests := viper.GetInt(maxConcurrentTestsFlagName)
@@ -94,19 +91,18 @@ func runE2E(ctx context.Context) error {
 	branchName := viper.GetString(branchNameFlagName)
 
 	runConf := e2e.ParallelRunConf{
-		MaxInstances:        maxInstances,
-		MaxConcurrentTests:  maxConcurrentTests,
-		AmiId:               amiId,
-		InstanceProfileName: instanceProfileName,
-		StorageBucket:       storageBucket,
-		JobId:               jobId,
-		SubnetId:            subnetId,
-		Regex:               testRegex,
-		TestsToSkip:         testsToSkip,
-		BundlesOverride:     bundlesOverride,
-		CleanupVms:          cleanupVms,
-		TestReportFolder:    testReportFolder,
-		BranchName:          branchName,
+		MaxInstances:           maxInstances,
+		MaxConcurrentTests:     maxConcurrentTests,
+		InstanceProfileName:    instanceProfileName,
+		StorageBucket:          storageBucket,
+		JobId:                  jobId,
+		Regex:                  testRegex,
+		TestsToSkip:            testsToSkip,
+		BundlesOverride:        bundlesOverride,
+		CleanupVms:             cleanupVms,
+		TestReportFolder:       testReportFolder,
+		BranchName:             branchName,
+		TestInstanceConfigFile: instanceConfigFile,
 	}
 
 	err := e2e.RunTestsInParallel(runConf)

go.mod

@@ -35,6 +35,7 @@ require (
 	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
 	gopkg.in/ini.v1 v1.66.2
 	gopkg.in/square/go-jose.v2 v2.6.0
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.23.4
 	k8s.io/apimachinery v0.23.4
@@ -71,6 +72,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/bmc-toolbox/bmclib v0.5.3 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/containerd/containerd v1.6.1 // indirect
 	github.com/coredns/caddy v1.1.0 // indirect
@@ -89,6 +91,8 @@ require (
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/go-playground/locales v0.13.0 // indirect
+	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/gobuffalo/flect v0.2.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.0.0 // indirect
@@ -100,16 +104,19 @@ require (
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/itchyny/gojq v0.12.6 // indirect
 	github.com/itchyny/timefmt-go v0.1.3 // indirect
+	github.com/jacobweinstock/registrar v0.4.6 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kevinburke/ssh_config v1.1.0 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/matryer/is v1.4.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
@@ -134,6 +141,7 @@ require (
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/stmcginnis/gofish v0.12.1-0.20220311113027-6072260f4c8d // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
@@ -147,9 +155,9 @@ require (
 	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
 	google.golang.org/grpc v1.43.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/go-playground/validator.v9 v9.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/apiextensions-apiserver v0.23.4 // indirect
 	k8s.io/cluster-bootstrap v0.23.0 // indirect
 	k8s.io/component-base v0.23.4 // indirect