Skip to content

Commit

Permalink
add tinkerbell ci test harness to e2e test framework
Browse files Browse the repository at this point in the history
  • Loading branch information
gwesterfieldjr committed May 10, 2022
1 parent 6a20e4b commit 1e8dcbc
Show file tree
Hide file tree
Showing 12 changed files with 437 additions and 75 deletions.
2 changes: 1 addition & 1 deletion cmd/integration_test/build/buildspecs/test-eks-a-cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ phases:
-i ${INTEGRATION_TEST_INSTANCE_PROFILE}
-n ${INTEGRATION_TEST_SUBNET_ID}
-m ${INTEGRATION_TEST_MAX_EC2_COUNT}
-c ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
-p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
-r 'Test'
-v 4
--skip ${SKIPPED_TESTS}
Expand Down
16 changes: 16 additions & 0 deletions cmd/integration_test/build/buildspecs/test-infra.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
---

ec2:
amiId: ami-00c71c2441b729da0
subnetId: subnet-0fd4b6ba7c9e17b5f

vSphere:
url: https://10.61.250.74
insecure: True
library: eks-a-templates
template: eks-a-admin-v0.0.0-e920823-20220509234801
datacenter: Datacenter
datastore: datastore2
resourcePool: TestResourcePool
network: VM Network
folder: gwesterf
42 changes: 19 additions & 23 deletions cmd/integration_test/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,9 @@ import (
)

const (
amiIdFlagName = "ami-id"
storageBucketFlagName = "storage-bucket"
jobIdFlagName = "job-id"
instanceProfileFlagName = "instance-profile-name"
subnetIdFlagName = "subnet-id"
regexFlagName = "regex"
maxInstancesFlagName = "max-instances"
maxConcurrentTestsFlagName = "max-concurrent-tests"
Expand All @@ -27,6 +25,7 @@ const (
cleanupVmsFlagName = "cleanup-vms"
testReportFolderFlagName = "test-report-folder"
branchNameFlagName = "branch-name"
instanceConfigFlagName = "instance-config"
)

var runE2ECmd = &cobra.Command{
Expand All @@ -44,7 +43,7 @@ var runE2ECmd = &cobra.Command{
},
}

var requiredFlags = []string{amiIdFlagName, storageBucketFlagName, jobIdFlagName, instanceProfileFlagName}
var requiredFlags = []string{instanceConfigFlagName, storageBucketFlagName, jobIdFlagName, instanceProfileFlagName}

func preRunSetup(cmd *cobra.Command, args []string) {
cmd.Flags().VisitAll(func(flag *pflag.Flag) {
Expand All @@ -57,18 +56,17 @@ func preRunSetup(cmd *cobra.Command, args []string) {

func init() {
integrationTestCmd.AddCommand(runE2ECmd)
runE2ECmd.Flags().StringP(amiIdFlagName, "a", "", "Ami id")
runE2ECmd.Flags().StringP(instanceConfigFlagName, "c", "", "File path to the instance-config.yml config")
runE2ECmd.Flags().StringP(storageBucketFlagName, "s", "", "S3 bucket name to store eks-a binary")
runE2ECmd.Flags().StringP(jobIdFlagName, "j", "", "Id of the job being run")
runE2ECmd.Flags().StringP(instanceProfileFlagName, "i", "", "IAM instance profile name to attach to ec2 instances")
runE2ECmd.Flags().StringP(subnetIdFlagName, "n", "", "EC2 subnet ID")
runE2ECmd.Flags().StringP(instanceProfileFlagName, "i", "", "IAM instance profile name to attach to ssm instances")
runE2ECmd.Flags().StringP(regexFlagName, "r", "", "Run only those tests and examples matching the regular expression. Equivalent to go test -run")
runE2ECmd.Flags().IntP(maxInstancesFlagName, "m", 1, "Run tests in parallel on same instance within the max EC2 instance count")
runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "c", 1, "Maximum number of parallel tests that can be run at a time")
runE2ECmd.Flags().IntP(maxConcurrentTestsFlagName, "p", 1, "Maximum number of parallel tests that can be run at a time")
runE2ECmd.Flags().StringSlice(skipFlagName, nil, "List of tests to skip")
runE2ECmd.Flags().Bool(bundlesOverrideFlagName, false, "Flag to indicate if the tests should run with a bundles override")
runE2ECmd.Flags().Bool(cleanupVmsFlagName, false, "Flag to indicate if VSphere VMs should be cleaned up automatically as tests complete")
runE2ECmd.Flags().String(testReportFolderFlagName, "", "Folder destination fo JUnit tests reports")
runE2ECmd.Flags().String(testReportFolderFlagName, "", "Folder destination for JUnit tests reports")
runE2ECmd.Flags().String(branchNameFlagName, "main", "EKS-A origin branch from where the tests are being run")

for _, flag := range requiredFlags {
Expand All @@ -79,11 +77,10 @@ func init() {
}

func runE2E(ctx context.Context) error {
amiId := viper.GetString(amiIdFlagName)
instanceConfigFile := viper.GetString(instanceConfigFlagName)
storageBucket := viper.GetString(storageBucketFlagName)
jobId := viper.GetString(jobIdFlagName)
instanceProfileName := viper.GetString(instanceProfileFlagName)
subnetId := viper.GetString(subnetIdFlagName)
testRegex := viper.GetString(regexFlagName)
maxInstances := viper.GetInt(maxInstancesFlagName)
maxConcurrentTests := viper.GetInt(maxConcurrentTestsFlagName)
Expand All @@ -94,19 +91,18 @@ func runE2E(ctx context.Context) error {
branchName := viper.GetString(branchNameFlagName)

runConf := e2e.ParallelRunConf{
MaxInstances: maxInstances,
MaxConcurrentTests: maxConcurrentTests,
AmiId: amiId,
InstanceProfileName: instanceProfileName,
StorageBucket: storageBucket,
JobId: jobId,
SubnetId: subnetId,
Regex: testRegex,
TestsToSkip: testsToSkip,
BundlesOverride: bundlesOverride,
CleanupVms: cleanupVms,
TestReportFolder: testReportFolder,
BranchName: branchName,
MaxInstances: maxInstances,
MaxConcurrentTests: maxConcurrentTests,
InstanceProfileName: instanceProfileName,
StorageBucket: storageBucket,
JobId: jobId,
Regex: testRegex,
TestsToSkip: testsToSkip,
BundlesOverride: bundlesOverride,
CleanupVms: cleanupVms,
TestReportFolder: testReportFolder,
BranchName: branchName,
TestInstanceConfigFile: instanceConfigFile,
}

err := e2e.RunTestsInParallel(runConf)
Expand Down
30 changes: 30 additions & 0 deletions internal/pkg/ssm/activation.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package ssm

import (
"fmt"

"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ssm"
)

type ActivationInfo struct {
ActivationCode string
ActivationID string
}

func CreateActivation(session *session.Session, instanceName, role string) (*ActivationInfo, error) {
s := ssm.New(session)

request := ssm.CreateActivationInput{
DefaultInstanceName: &instanceName,
Description: &instanceName,
IamRole: &role,
}

result, err := s.CreateActivation(&request)
if err != nil {
return nil, fmt.Errorf("failed to activate ssm instance %s: %v", instanceName, err)
}

return &ActivationInfo{ActivationCode: *result.ActivationCode, ActivationID: *result.ActivationId}, nil
}
31 changes: 31 additions & 0 deletions internal/pkg/ssm/instance.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package ssm

import (
"fmt"

"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ssm"
)

func GetInstanceByActivationId(session *session.Session, id string) (*ssm.InstanceInformation, error) {
s := ssm.New(session)
instanceActivationIdKey := "ActivationIds"
input := ssm.DescribeInstanceInformationInput{
Filters: []*ssm.InstanceInformationStringFilter{
{Key: &instanceActivationIdKey, Values: []*string{&id}},
},
}

output, err := s.DescribeInstanceInformation(&input)
if err != nil {
return nil, fmt.Errorf("failed to describe ssm instance %s: %v", id, err)
}

infoList := output.InstanceInformationList

if len(infoList) == 0 {
return nil, fmt.Errorf("no ssm instance with name %s: %v", id, err)
}

return infoList[0], nil
}
56 changes: 56 additions & 0 deletions internal/pkg/vsphere/deploy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package vsphere

import (
"context"
"encoding/json"
"fmt"

"github.com/aws/eks-anywhere/pkg/executables"
"github.com/aws/eks-anywhere/pkg/filewriter"
)

type OVFDeployOptions struct {
Name string `json:"Name"`
PowerOn bool `json:"PowerOn"`
DiskProvisioning string `json:"DiskProvisioning"`
WaitForIP bool `json:"WaitForIP"`
NetworkMappings []NetworkMapping `json:"NetworkMapping"`
Annotation string `json:"Annotation"`
PropertyMapping []OVFProperty `json:"PropertyMapping"`
InjectOvfEnv bool `json:"InjectOvfEnv"`
}

type OVFProperty struct {
Key string `json:"Key"`
Value string `json:"Value"`
}

type NetworkMapping struct {
Name string `json:"Name"`
Network string `json:"Network"`
}

func DeployTemplate(library, templateName, vmName, deployFolder, datacenter, datastore, resourcePool string, opts OVFDeployOptions) error {
context := context.Background()
executableBuilder, close, err := executables.NewExecutableBuilder(context, executables.DefaultEksaImage())
if err != nil {
return fmt.Errorf("unable to initialize executables: %v", err)
}

defer close.CheckErr(context)
tmpWriter, _ := filewriter.NewWriter(vmName)
govc := executableBuilder.BuildGovcExecutable(tmpWriter)
defer govc.Close(context)

deployOptions, err := json.Marshal(opts)
if err != nil {
return fmt.Errorf("failed to marshall vm deployment options: %v", err)
}

// deploy template
if err := govc.DeployTemplate(context, library, templateName, vmName, deployFolder, datacenter, datastore, resourcePool, deployOptions); err != nil {
return fmt.Errorf("failed to deploy vm from library template: %v", err)
}

return nil
}
76 changes: 53 additions & 23 deletions internal/test/e2e/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strings"
"sync"

"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/eks-anywhere/internal/pkg/ec2"
"github.com/aws/eks-anywhere/internal/pkg/ssm"
"github.com/aws/eks-anywhere/pkg/logger"
Expand All @@ -23,19 +24,18 @@ const (
)

type ParallelRunConf struct {
MaxInstances int
MaxConcurrentTests int
AmiId string
InstanceProfileName string
StorageBucket string
JobId string
SubnetId string
Regex string
TestsToSkip []string
BundlesOverride bool
CleanupVms bool
TestReportFolder string
BranchName string
TestInstanceConfigFile string
MaxInstances int
MaxConcurrentTests int
InstanceProfileName string
StorageBucket string
JobId string
Regex string
TestsToSkip []string
BundlesOverride bool
CleanupVms bool
TestReportFolder string
BranchName string
}

type (
Expand Down Expand Up @@ -63,7 +63,11 @@ func RunTestsInParallel(conf ParallelRunConf) error {

var wg sync.WaitGroup

instancesConf := splitTests(testsList, conf)
instancesConf, err := splitTests(testsList, conf)
if err != nil {
return fmt.Errorf("failed to split tests: %v", err)
}

results := make([]instanceTestsResults, 0, len(instancesConf))
logTestGroups(instancesConf)
maxConcurrentTests := conf.MaxConcurrentTests
Expand Down Expand Up @@ -119,14 +123,23 @@ func RunTestsInParallel(conf ParallelRunConf) error {
}

type instanceRunConf struct {
amiId, instanceProfileName, storageBucket, jobId, parentJobId, subnetId, regex, instanceId string
testReportFolder, branchName string
ipPool networkutils.IPPool
bundlesOverride bool
session *session.Session
instanceProfileName, storageBucket, jobId, parentJobId, regex, instanceId string
testReportFolder, branchName string
ipPool networkutils.IPPool
bundlesOverride bool
testRunnerType TestRunnerType
testRunnerConfig TestInfraConfig
}

func RunTests(conf instanceRunConf) (testInstanceID string, testCommandResult *testCommandResult, err error) {
session, err := newSessionFromConf(conf)
testRunner := newTestRunner(conf.testRunnerType, conf.testRunnerConfig)
instanceId, err := testRunner.createInstance(conf)
if err != nil {
return "", nil, err
}

session, err := newE2ESession(instanceId, conf)
if err != nil {
return "", nil, err
}
Expand Down Expand Up @@ -207,13 +220,14 @@ func (e *E2ESession) commandWithEnvVars(command string) string {
return strings.Join(fullCommand, "; ")
}

func splitTests(testsList []string, conf ParallelRunConf) []instanceRunConf {
func splitTests(testsList []string, conf ParallelRunConf) ([]instanceRunConf, error) {
testPerInstance := len(testsList) / conf.MaxInstances
if testPerInstance == 0 {
testPerInstance = 1
}

vsphereTestsRe := regexp.MustCompile(vsphereRegex)
tinkerbellTestsRe := regexp.MustCompile(tinkerbellTestsRe)
privateNetworkTestsRe := regexp.MustCompile(`^.*(Proxy|RegistryMirror).*$`)
multiClusterTestsRe := regexp.MustCompile(`^.*Multicluster.*$`)

Expand All @@ -239,26 +253,42 @@ func splitTests(testsList []string, conf ParallelRunConf) []instanceRunConf {
}
}

awsSession, err := session.NewSession()
if err != nil {
return nil, fmt.Errorf("creating aws session for test: %s, %v", testName, err)
}

testRunnerConfig, err := NewTestRunnerConfigFromFile(conf.TestInstanceConfigFile)
if err != nil {
return nil, fmt.Errorf("creating test runner config for test: %s, %v", testName, err)
}

testRunnerType := Ec2TestRunnerType
if tinkerbellTestsRe.MatchString(testName) {
testRunnerType = VSphereTestRunnerType
}

if len(testsInCurrentInstance) == testPerInstance || (len(testsList)-1) == i {
runConfs = append(runConfs, instanceRunConf{
amiId: conf.AmiId,
session: awsSession,
instanceProfileName: conf.InstanceProfileName,
storageBucket: conf.StorageBucket,
jobId: fmt.Sprintf("%s-%d", conf.JobId, len(runConfs)),
parentJobId: conf.JobId,
subnetId: conf.SubnetId,
regex: strings.Join(testsInCurrentInstance, "|"),
bundlesOverride: conf.BundlesOverride,
testReportFolder: conf.TestReportFolder,
branchName: conf.BranchName,
ipPool: ips,
testRunnerType: testRunnerType,
testRunnerConfig: *testRunnerConfig,
})

testsInCurrentInstance = make([]string, 0, testPerInstance)
}
}

return runConfs
return runConfs, nil
}

func logTestGroups(instancesConf []instanceRunConf) {
Expand Down
Loading

0 comments on commit 1e8dcbc

Please sign in to comment.