Support Cloudstack multiple endpoint for preflight checks (#2559)

* Support Cloudstack multiple endpoint for preflight checks

* Update for review comments

* Fix unit test failures

* Remove debug print

* Remove unnecessary file

* Remove multierror module

* Update to use single cmk object instead of list or map of multiple cmks

* Remove ValidateCloudStackConnection when cleaning up VMs

* Remove NewProviderCustomNet because it's the same as NewProvider

* Avoid using 'failed to' in errors

* Remove localAvailabilityZones from Validator

* Remove multipleZone from ValidateNetworkPresent

* Return list of errors CleanUpCloudstackTestResources

* Increase unit test coverage

* Increase unit test coverage #2

* Change error format

* Change function names to be more descriptive
- ValidateZonePresent -> ValidateZoneAndGetId
- ValidateDomainPresent -> ValidateDomainAndGetId
Change return value of ValidateDomainAndGetId
- from CloudStackResourceIdentifier to string because only domainId is used

* Add default verifySslValue as a constant

* Fix a bug where empty Zone IDs are used in validateMachineConfig

* Add error message when validating management api endpoint fails

* Make MarkPass message more readable

* Fix presubmit job failure

internal/test/cleanup/cleanup.go

@@ -99,12 +99,18 @@ func CleanUpCloudstackTestResources(ctx context.Context, clusterName string, dry
 	if err != nil {
 		return fmt.Errorf("building cmk executable: %v", err)
 	}
-	cmk := executableBuilder.BuildCmkExecutable(tmpWriter, *execConfig)
+	cmk := executableBuilder.BuildCmkExecutable(tmpWriter, execConfig.Profiles)
 	defer cmk.Close(ctx)
 
-	if err := cmk.ValidateCloudStackConnection(ctx); err != nil {
-		return fmt.Errorf("validating cloudstack connection with cloudmonkey: %v", err)
+	errorsMap := map[string]error{}
+	for _, profile := range execConfig.Profiles {
+		if err := cmk.CleanupVms(ctx, profile.Name, clusterName, dryRun); err != nil {
+			errorsMap[profile.Name] = err
+		}
 	}
 
-	return cmk.CleanupVms(ctx, clusterName, dryRun)
+	if len(errorsMap) > 0 {
+		return fmt.Errorf("cleaning up VMs: %+v", errorsMap)
+	}
+	return nil
 }

pkg/dependencies/factory.go

@@ -418,12 +418,13 @@ func (f *Factory) WithCmk() *Factory {
 		if f.dependencies.Cmk != nil {
 			return nil
 		}
+
 		execConfig, err := decoder.ParseCloudStackSecret()
 		if err != nil {
 			return fmt.Errorf("building cmk executable: %v", err)
 		}
 
-		f.dependencies.Cmk = f.executableBuilder.BuildCmkExecutable(f.dependencies.Writer, *execConfig)
+		f.dependencies.Cmk = f.executableBuilder.BuildCmkExecutable(f.dependencies.Writer, execConfig.Profiles)
 		f.dependencies.closers = append(f.dependencies.closers, f.dependencies.Cmk)
 
 		return nil

pkg/dependencies/factory_test.go

@@ -2,6 +2,7 @@ package dependencies_test
 
 import (
 	"context"
+	"encoding/base64"
 	"os"
 	"testing"
 
@@ -13,6 +14,7 @@ import (
 	"github.com/aws/eks-anywhere/pkg/cluster"
 	"github.com/aws/eks-anywhere/pkg/config"
 	"github.com/aws/eks-anywhere/pkg/dependencies"
+	"github.com/aws/eks-anywhere/pkg/providers/cloudstack/decoder"
 	"github.com/aws/eks-anywhere/pkg/version"
 	"github.com/aws/eks-anywhere/release/api/v1alpha1"
 )
@@ -107,6 +109,10 @@ func TestFactoryBuildWithClusterManagerWithoutCliConfig(t *testing.T) {
 }
 
 func TestFactoryBuildWithMultipleDependencies(t *testing.T) {
+	configString := test.ReadFile(t, "testdata/cloudstack_config_multiple_profiles.ini")
+	encodedConfig := base64.StdEncoding.EncodeToString([]byte(configString))
+	t.Setenv(decoder.EksacloudStackCloudConfigB64SecretKey, encodedConfig)
+
 	tt := newTest(t, vsphere)
 	deps, err := dependencies.NewFactory().
 		UseExecutableImage("image:1").
@@ -125,6 +131,7 @@ func TestFactoryBuildWithMultipleDependencies(t *testing.T) {
 		WithCAPIManager().
 		WithManifestReader().
 		WithUnAuthKubeClient().
+		WithCmk().
 		Build(context.Background())
 
 	tt.Expect(err).To(BeNil())

pkg/dependencies/testdata/cloudstack_config_multiple_profiles.ini

@@ -0,0 +1,11 @@
+[Global]
+verify-ssl = false
+api-key = test-key1
+secret-key = test-secret1
+api-url = http://127.16.0.1:8080/client/api
+
+[Instance2]
+verify-ssl = true
+api-key = test-key2
+secret-key = test-secret2
+api-url = http://127.16.0.2:8080/client/api

pkg/executables/builder.go

@@ -41,8 +41,8 @@ func (b *ExecutableBuilder) BuildGovcExecutable(writer filewriter.FileWriter, op
 	return NewGovc(b.buildExecutable(govcPath), writer, opts...)
 }
 
-func (b *ExecutableBuilder) BuildCmkExecutable(writer filewriter.FileWriter, execConfig decoder.CloudStackExecConfig) *Cmk {
-	return NewCmk(b.buildExecutable(cmkPath), writer, execConfig)
+func (b *ExecutableBuilder) BuildCmkExecutable(writer filewriter.FileWriter, configs []decoder.CloudStackProfileConfig) *Cmk {
+	return NewCmk(b.buildExecutable(cmkPath), writer, configs)
 }
 
 func (b *ExecutableBuilder) BuildAwsCli() *AwsCli {