diff --git a/config/crd/bases/anywhere.eks.amazonaws.com_cloudstackdatacenterconfigs.yaml b/config/crd/bases/anywhere.eks.amazonaws.com_cloudstackdatacenterconfigs.yaml index ec9347b632d5..8087a29a2f59 100644 --- a/config/crd/bases/anywhere.eks.amazonaws.com_cloudstackdatacenterconfigs.yaml +++ b/config/crd/bases/anywhere.eks.amazonaws.com_cloudstackdatacenterconfigs.yaml @@ -46,6 +46,85 @@ spec: on resources within that account. If an account name is provided, a domain must also be provided. type: string + availabilityZones: + description: AvailabilityZones list of different partitions to distribute + VMs across - corresponds to a list of CAPI failure domains + items: + description: CloudStackAvailabilityZone maps to a CAPI failure domain + to distribute machines across Cloudstack infrastructure + properties: + account: + description: Account typically represents a customer of the + service provider or a department in a large organization. + Multiple users can exist in an account, and all CloudStack + resources belong to an account. Accounts have users and users + have credentials to operate on resources within that account. + If an account name is provided, a domain must also be provided. + type: string + credentialsRef: + description: CredentialRef is used to reference a secret in + the eksa-system namespace + type: string + domain: + description: Domain contains a grouping of accounts. Domains + usually contain multiple accounts that have some logical relationship + to each other and a set of delegated administrators with some + authority over the domain and its subdomains This field is + considered as a fully qualified domain name which is the same + as the domain path without "ROOT/" prefix. For example, if + "foo" is specified then a domain with "ROOT/foo" domain path + is picked. The value "ROOT" is a special case that points + to "the" ROOT domain of the CloudStack. That is, a domain + with a path "ROOT/ROOT" is not allowed. + type: string + managementApiEndpoint: + description: CloudStack Management API endpoint's IP. It is + added to VM's noproxy list + type: string + name: + description: Name is used as a unique identifier for each availability + zone + type: string + zone: + description: Zone represents the properties of the CloudStack + zone in which clusters should be created, like the network. + properties: + id: + description: Zone is the name or UUID of the CloudStack + zone in which clusters should be created. Zones should + be managed by a single CloudStack Management endpoint. + type: string + name: + type: string + network: + description: Network is the name or UUID of the CloudStack + network in which clusters should be created. It can either + be an isolated or shared network. If it doesn’t already + exist in CloudStack, it’ll automatically be created by + CAPC as an isolated network. It can either be specified + as a UUID or name In multiple-zones situation, only 'Shared' + network is supported. + properties: + id: + description: Id of a resource in the CloudStack environment. + Mutually exclusive with Name + type: string + name: + description: Name of a resource in the CloudStack environment. + Mutually exclusive with Id + type: string + type: object + required: + - network + type: object + required: + - credentialsRef + - domain + - managementApiEndpoint + - name + - zone + type: object + type: array domain: description: Domain contains a grouping of accounts. Domains usually contain multiple accounts that have some logical relationship to @@ -98,10 +177,6 @@ spec: - network type: object type: array - required: - - domain - - managementApiEndpoint - - zones type: object status: description: CloudStackDatacenterConfigStatus defines the observed state diff --git a/config/manifest/eksa-components.yaml b/config/manifest/eksa-components.yaml index 56db675e7e81..dbca7fea1d17 100644 --- a/config/manifest/eksa-components.yaml +++ b/config/manifest/eksa-components.yaml @@ -3303,6 +3303,85 @@ spec: on resources within that account. If an account name is provided, a domain must also be provided. type: string + availabilityZones: + description: AvailabilityZones list of different partitions to distribute + VMs across - corresponds to a list of CAPI failure domains + items: + description: CloudStackAvailabilityZone maps to a CAPI failure domain + to distribute machines across Cloudstack infrastructure + properties: + account: + description: Account typically represents a customer of the + service provider or a department in a large organization. + Multiple users can exist in an account, and all CloudStack + resources belong to an account. Accounts have users and users + have credentials to operate on resources within that account. + If an account name is provided, a domain must also be provided. + type: string + credentialsRef: + description: CredentialRef is used to reference a secret in + the eksa-system namespace + type: string + domain: + description: Domain contains a grouping of accounts. Domains + usually contain multiple accounts that have some logical relationship + to each other and a set of delegated administrators with some + authority over the domain and its subdomains This field is + considered as a fully qualified domain name which is the same + as the domain path without "ROOT/" prefix. For example, if + "foo" is specified then a domain with "ROOT/foo" domain path + is picked. The value "ROOT" is a special case that points + to "the" ROOT domain of the CloudStack. That is, a domain + with a path "ROOT/ROOT" is not allowed. + type: string + managementApiEndpoint: + description: CloudStack Management API endpoint's IP. It is + added to VM's noproxy list + type: string + name: + description: Name is used as a unique identifier for each availability + zone + type: string + zone: + description: Zone represents the properties of the CloudStack + zone in which clusters should be created, like the network. + properties: + id: + description: Zone is the name or UUID of the CloudStack + zone in which clusters should be created. Zones should + be managed by a single CloudStack Management endpoint. + type: string + name: + type: string + network: + description: Network is the name or UUID of the CloudStack + network in which clusters should be created. It can either + be an isolated or shared network. If it doesn’t already + exist in CloudStack, it’ll automatically be created by + CAPC as an isolated network. It can either be specified + as a UUID or name In multiple-zones situation, only 'Shared' + network is supported. + properties: + id: + description: Id of a resource in the CloudStack environment. + Mutually exclusive with Name + type: string + name: + description: Name of a resource in the CloudStack environment. + Mutually exclusive with Id + type: string + type: object + required: + - network + type: object + required: + - credentialsRef + - domain + - managementApiEndpoint + - name + - zone + type: object + type: array domain: description: Domain contains a grouping of accounts. Domains usually contain multiple accounts that have some logical relationship to @@ -3355,10 +3434,6 @@ spec: - network type: object type: array - required: - - domain - - managementApiEndpoint - - zones type: object status: description: CloudStackDatacenterConfigStatus defines the observed state diff --git a/designs/cloudstack-multiple-endpoints.md b/designs/cloudstack-multiple-endpoints.md index f24a30d02deb..ce2a5c6afd8f 100644 --- a/designs/cloudstack-multiple-endpoints.md +++ b/designs/cloudstack-multiple-endpoints.md @@ -47,52 +47,54 @@ will need to be available on *all* the Cloudstack API endpoints. We will validat ### Interface changes Currently, the CloudstackDataCenterConfig spec contains: -``` +```go type CloudStackDatacenterConfigSpec struct { - // Domain contains a grouping of accounts. Domains usually contain multiple accounts that have some logical relationship to each other and a set of delegated administrators with some authority over the domain and its subdomains - Domain string `json:"domain"` - // Zones is a list of one or more zones that are managed by a single CloudStack management endpoint. - Zones []CloudStackZone `json:"zones"` - // Account typically represents a customer of the service provider or a department in a large organization. Multiple users can exist in an account, and all CloudStack resources belong to an account. Accounts have users and users have credentials to operate on resources within that account. If an account name is provided, a domain must also be provided. - Account string `json:"account,omitempty"` - // CloudStack Management API endpoint's IP. It is added to VM's noproxy list - ManagementApiEndpoint string `json:"managementApiEndpoint"` + // Domain contains a grouping of accounts. Domains usually contain multiple accounts that have some logical relationship to each other and a set of delegated administrators with some authority over the domain and its subdomains + Domain string `json:"domain"` + // Zones is a list of one or more zones that are managed by a single CloudStack management endpoint. + Zones []CloudStackZone `json:"zones"` + // Account typically represents a customer of the service provider or a department in a large organization. Multiple users can exist in an account, and all CloudStack resources belong to an account. Accounts have users and users have credentials to operate on resources within that account. If an account name is provided, a domain must also be provided. + Account string `json:"account,omitempty"` + // CloudStack Management API endpoint's IP. It is added to VM's noproxy list + ManagementApiEndpoint string `json:"managementApiEndpoint"` } ``` We would instead propose to gradually deprecate all the existing attributes and instead, simply include a list of AvailabilityZone objects like so -``` +```go type CloudStackDatacenterConfigSpec struct { - // Deprecated - Domain string `json:"domain,omitempty"` - // Deprecated - Zones []CloudStackZone `json:"zones,omitempty"` - // Deprecated - Account string `json:"account,omitempty"` - // Deprecated - ManagementApiEndpoint string `json:"managementApiEndpoint,omitempty"` - // List of AvailabilityZones to distribute VMs across - corresponds to a list of CAPI failure domains - AvailabilityZones []CloudStackAvailabilityZone `json:"availabilityZones,omitempty"` + // Deprecated + Domain string `json:"domain,omitempty"` + // Deprecated + Zones []CloudStackZone `json:"zones,omitempty"` + // Deprecated + Account string `json:"account,omitempty"` + // Deprecated + ManagementApiEndpoint string `json:"managementApiEndpoint,omitempty"` + // List of AvailabilityZones to distribute VMs across - corresponds to a list of CAPI failure domains + AvailabilityZones []CloudStackAvailabilityZone `json:"availabilityZones,omitempty"` } ``` where each AvailabilityZone object looks like -``` +```go type CloudStackAvailabilityZone struct { - // Name would be used to match the availability zone defined in the datacenter config to the credentials passed in from the cloud-config ini file - Name string `json:"name"` - // Domain contains a grouping of accounts. Domains usually contain multiple accounts that have some logical relationship to each other and a set of delegated administrators with some authority over the domain and its subdomains - // This field is considered as a fully qualified domain name which is the same as the domain path without "ROOT/" prefix. For example, if "foo" is specified then a domain with "ROOT/foo" domain path is picked. - // The value "ROOT" is a special case that points to "the" ROOT domain of the CloudStack. That is, a domain with a path "ROOT/ROOT" is not allowed. - Domain string `json:"domain"` - // Zones is a list of one or more zones that are managed by a single CloudStack management endpoint. - Zone CloudStackZone `json:"zone"` - // Account typically represents a customer of the service provider or a department in a large organization. Multiple users can exist in an account, and all CloudStack resources belong to an account. Accounts have users and users have credentials to operate on resources within that account. If an account name is provided, a domain must also be provided. - Account string `json:"account,omitempty"` - // CloudStack Management API endpoint's IP. It is added to VM's noproxy list - ManagementApiEndpoint string `json:"managementApiEndpoint"` + // Name would be used as a unique identifier for each availability zone + Name string `json:"name"` + // CredentialRef would be used to match a secret in the eksa-system namespace + CredentialsRef string `json:"credentialsRef" + // Domain contains a grouping of accounts. Domains usually contain multiple accounts that have some logical relationship to each other and a set of delegated administrators with some authority over the domain and its subdomains + // This field is considered as a fully qualified domain name which is the same as the domain path without "ROOT/" prefix. For example, if "foo" is specified then a domain with "ROOT/foo" domain path is picked. + // The value "ROOT" is a special case that points to "the" ROOT domain of the CloudStack. That is, a domain with a path "ROOT/ROOT" is not allowed. + Domain string `json:"domain"` + // Zones is a list of one or more zones that are managed by a single CloudStack management endpoint. + Zone CloudStackZone `json:"zone"` + // Account typically represents a customer of the service provider or a department in a large organization. Multiple users can exist in an account, and all CloudStack resources belong to an account. Accounts have users and users have credentials to operate on resources within that account. If an account name is provided, a domain must also be provided. + Account string `json:"account,omitempty"` + // CloudStack Management API endpoint's IP. It is added to VM's noproxy list + ManagementApiEndpoint string `json:"managementApiEndpoint"` } ``` @@ -107,7 +109,7 @@ CAPC currently utilizes them to distribute machines across CloudStack Zones. How 2. Cloudstack domain 3. Cloudstack zone 4. Cloudstack account -5. A unique name +5. A reference to the customer-provider credentials for interacting with this Cloudstack endpoint You can find more information about these Cloudstack resources [here](http://docs.cloudstack.apache.org/en/latest/conceptsandterminology/concepts.html#cloudstack-terminology) @@ -117,6 +119,9 @@ With the multi-endpoint system for the Cloudstack provider, users reference a Cl is that all the Cloudstack resources such as image, ComputeOffering, ISOAttachment, etc. must be available in *all* the AvailabilityZones, or all the Cloudstack endpoints, and these resources must be referenced by name, not unique ID. This would mean that we need to check if there are multiple Cloudstack endpoints, and if so check the zones, networks, domains, accounts, and users. +We will also validate the credentials referenced by each AvailabilityZone, which can either be referenced as existing K8s secrets on the cluster, or from the local ini file. More details +in the Cloudstack Credentials section below + ### `CloudstackMachineConfig` Validation For each CloudstackMachineConfig, we have to make sure that all the prerequisite @@ -129,13 +134,12 @@ for availabilityZone in availabilityZones: validate resource presence with the availabilityZone's configuration of the CloudMonkey executable -### Cloudstack credentials - +### Cloudstack Credentials In a multi-endpoint Cloudstack cluster, each endpoint may have its own credentials. We propose that Cloudstack credentials will be passed in via environment variable in the same way as they are currently, -only as a list corresponding to AvailabilityZones. Currently, these credentials are passed in via environment variable, which contains a base64 encoded .ini file that looks like +only as a list corresponding to some K8s secrets which will be generated. Currently, these credentials are passed in via environment variable, which contains a base64 encoded .ini file that looks like -``` +```ini [Global] api-key = redacted secret-key = redacted @@ -144,7 +148,7 @@ api-url = http://172.16.0.1:8080/client/api We would propose an extension of the above input mechanism so the user could provide credentials across multiple Cloudstack API endpoints like -``` +```ini [Global] api-key = redacted secret-key = redacted @@ -163,10 +167,26 @@ api-url = http://172.16.0.3:8080/client/api ... ``` -Where the Section names (i.e. Global, AvailabilityZone1, etc.) correspond to the Availability Zone names +Where the Section names (i.e. Global, AvailabilityZone1, etc.) correspond to the credentials needed to access a given Availability Zone. -We are also exploring converting the ini file to a yaml input file which contains a list of credentials and their associated endpoints. Either way, this environment variable would -be passed along to CAPC and used by the CAPC controller just like it is currently. +In order to avoid taking a dependency on a meaningless section of a transient file, we will modify EKS-A to also create the secrets to be used by CAPC. +These secrets will be generated from the ini file provided by customer and applied to the cluster. CAPC will then proceed to take ownership of them as +currently done in [CAPV](https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/fae6ef88467e608665e2902e2bb0aaeb4cee67ed/docs/identity_management.md#identity-types). +We will refer to this set of credentials in the CloudStackCluster resource as well so CAPC knows which ones to use. + +So for create, we’ll have: + +1. Customer provides ini file containing set of named credentials +2. Customer provides CloudstackDatacenterConfigSpec with list of named AvailabilityZones to distribute the cluster across. Each AvailabilityZone will have a credentialRef which +can either point to an existing secret on the cluster, or a named credential from the ini file +3. EKS-A will check each credentialRef in the AvailabilityZones provided. If it’s already in the cluster as a secret, validate it against the +contents in the ini file. If it’s not already in the cluster as a secret but there’s an entry in the ini file for it, create a new secret. If it’s +not in the cluster as a secret and not in the ini file, throw an error +4. Proceed to generate CAPC template and pass these secretRefs to CAPC who will add an OwnerRef to them + +The secrets will be managed by EKS-A insofar that they can be created but not updated. If users want to update an existing secret, they will have to +do so manually both in the K8s secret object, as well as the local ini file prior to running upgrade. We will also error out if there is a discrepancy in secret contents between the K8s object and the ini file. +This will provide a safeguard to prevent unintentionally setting incorrect credentials for a whole collection of clusters. ### Backwards Compatibility @@ -176,10 +196,8 @@ In order to support backwards compatibility in the CloudstackDatacenterConfig re Between these two approaches, we propose to take the first and then deprecate the legacy fields in a subsequent release to simplify the code paths. -However, given that the Cloudstack credentials are persisted in a write-once secret on the cluster, upgrading existing clusters may not be feasible unless CAPC supports overwriting that secret. - -## User Experience - +Upgrading an existing cluster will require passing the new credentials and templates to CAPC in a way that the mapping from CAPC FailureDomains to EKS-A AvailabilityZones can +be preserved. We plan to align on naming conventions for upgrading existing clusters based on some metadata from the current multi-zone configuration into a multi-endpoint configuration. ## Security @@ -187,6 +205,9 @@ The main change regarding security is the additional credential management. Othe create yaml templates and apply them, and then read/write eks-a resources in the eks-a controller. The corresponding change is an extension of an existing mechanism and there should not be any new surface area for risks than there was previously. +One risk to consider with regards to the new credential management strategy is that if AvailabilityZones are upgraded to use a new credential, there is a possibility of having old rogue Secret objects +which will need to be cleaned up manually. + ## Testing The new code will be covered by unit and e2e tests, and the e2e framework will be extended to support cluster creation across multiple Cloudstack API endpoints. diff --git a/pkg/api/v1alpha1/cloudstackdatacenterconfig_test.go b/pkg/api/v1alpha1/cloudstackdatacenterconfig_test.go index 4bb8ce654efe..5154549c7473 100644 --- a/pkg/api/v1alpha1/cloudstackdatacenterconfig_test.go +++ b/pkg/api/v1alpha1/cloudstackdatacenterconfig_test.go @@ -168,6 +168,24 @@ var cloudStackDatacenterConfigSpec1 = &CloudStackDatacenterConfigSpec{ ManagementApiEndpoint: "testEndpoint", } +var cloudStackDatacenterConfigSpecAzs = &CloudStackDatacenterConfigSpec{ + AvailabilityZones: []CloudStackAvailabilityZone{ + { + Name: "availability-zone-0", + CredentialsRef: "Global", + Zone: CloudStackZone{ + Name: "zone1", + Network: CloudStackResourceIdentifier{ + Name: "net1", + }, + }, + Account: "admin", + Domain: "domain1", + ManagementApiEndpoint: "testEndpoint", + }, + }, +} + func TestCloudStackDatacenterConfigSpecEqual(t *testing.T) { g := NewWithT(t) cloudStackDatacenterConfigSpec2 := cloudStackDatacenterConfigSpec1.DeepCopy() @@ -201,3 +219,64 @@ func TestCloudStackDatacenterConfigSpecNotEqualZonesNil(t *testing.T) { cloudStackDatacenterConfigSpec2.Zones = nil g.Expect(cloudStackDatacenterConfigSpec1.Equal(cloudStackDatacenterConfigSpec2)).To(BeFalse(), "Zones comparison in CloudStackDatacenterConfigSpec not detected") } + +func TestCloudStackDatacenterConfigSpecNotEqualAvailabilityZonesNil(t *testing.T) { + g := NewWithT(t) + g.Expect(cloudStackDatacenterConfigSpecAzs.AvailabilityZones[0].Equal(nil)).To(BeFalse(), "Zones comparison in CloudStackDatacenterConfigSpec not detected") +} + +func TestCloudStackDatacenterConfigSpecNotEqualAvailabilityZonesEmpty(t *testing.T) { + g := NewWithT(t) + cloudStackDatacenterConfigSpec2 := cloudStackDatacenterConfigSpecAzs.DeepCopy() + cloudStackDatacenterConfigSpec2.AvailabilityZones = []CloudStackAvailabilityZone{} + g.Expect(cloudStackDatacenterConfigSpecAzs.Equal(cloudStackDatacenterConfigSpec2)).To(BeFalse(), "AvailabilityZones comparison in CloudStackDatacenterConfigSpec not detected") +} + +func TestCloudStackDatacenterConfigSpecNotEqualAvailabilityZonesModified(t *testing.T) { + g := NewWithT(t) + cloudStackDatacenterConfigSpec2 := cloudStackDatacenterConfigSpecAzs.DeepCopy() + cloudStackDatacenterConfigSpec2.AvailabilityZones[0].Account = "differentAccount" + g.Expect(cloudStackDatacenterConfigSpec1.Equal(cloudStackDatacenterConfigSpec2)).To(BeFalse(), "AvailabilityZones comparison in CloudStackDatacenterConfigSpec not detected") +} + +func TestCloudStackAvailabilityZonesEqual(t *testing.T) { + g := NewWithT(t) + cloudStackAvailabilityZoneSpec2 := cloudStackDatacenterConfigSpecAzs.AvailabilityZones[0].DeepCopy() + g.Expect(cloudStackDatacenterConfigSpecAzs.AvailabilityZones[0].Equal(cloudStackAvailabilityZoneSpec2)).To(BeTrue(), "AvailabilityZones comparison in CloudStackAvailabilityZoneSpec not detected") +} + +func TestCloudStackAvailabilityZonesSame(t *testing.T) { + g := NewWithT(t) + g.Expect(cloudStackDatacenterConfigSpecAzs.AvailabilityZones[0].Equal(&cloudStackDatacenterConfigSpecAzs.AvailabilityZones[0])).To(BeTrue(), "AvailabilityZones comparison in CloudStackAvailabilityZoneSpec not detected") +} + +func TestCloudStackDatacenterConfigSpecNotEqualAvailabilityZonesManagementApiEndpoint(t *testing.T) { + g := NewWithT(t) + cloudStackDatacenterConfigSpec2 := cloudStackDatacenterConfigSpecAzs.DeepCopy() + cloudStackDatacenterConfigSpec2.AvailabilityZones[0].ManagementApiEndpoint = "fake-endpoint" + g.Expect(cloudStackDatacenterConfigSpec1.Equal(cloudStackDatacenterConfigSpec2)).To(BeFalse(), "AvailabilityZones comparison in CloudStackDatacenterConfigSpec not detected") +} + +func TestCloudStackDatacenterConfigSpecNotEqualAvailabilityZonesAccount(t *testing.T) { + g := NewWithT(t) + cloudStackDatacenterConfigSpec2 := cloudStackDatacenterConfigSpecAzs.DeepCopy() + cloudStackDatacenterConfigSpec2.AvailabilityZones[0].Account = "fake-acc" + g.Expect(cloudStackDatacenterConfigSpec1.Equal(cloudStackDatacenterConfigSpec2)).To(BeFalse(), "AvailabilityZones comparison in CloudStackDatacenterConfigSpec not detected") +} + +func TestCloudStackDatacenterConfigSpecNotEqualAvailabilityZonesDomain(t *testing.T) { + g := NewWithT(t) + cloudStackDatacenterConfigSpec2 := cloudStackDatacenterConfigSpecAzs.DeepCopy() + cloudStackDatacenterConfigSpec2.AvailabilityZones[0].Domain = "fake-domain" + g.Expect(cloudStackDatacenterConfigSpec1.Equal(cloudStackDatacenterConfigSpec2)).To(BeFalse(), "AvailabilityZones comparison in CloudStackDatacenterConfigSpec not detected") +} + +func TestCloudStackDatacenterConfigSetDefaults(t *testing.T) { + g := NewWithT(t) + cloudStackDatacenterConfig := CloudStackDatacenterConfig{ + Spec: *cloudStackDatacenterConfigSpec1.DeepCopy(), + } + cloudStackDatacenterConfig.SetDefaults() + g.Expect(cloudStackDatacenterConfig.Spec.Equal(cloudStackDatacenterConfigSpecAzs)).To(BeTrue(), "AvailabilityZones comparison in CloudStackDatacenterConfigSpec not equal") + g.Expect(len(cloudStackDatacenterConfigSpec1.Zones)).To(Equal(len(cloudStackDatacenterConfig.Spec.AvailabilityZones)), "AvailabilityZones count in CloudStackDatacenterConfigSpec not equal to zone count") +} diff --git a/pkg/api/v1alpha1/cloudstackdatacenterconfig_types.go b/pkg/api/v1alpha1/cloudstackdatacenterconfig_types.go index a2054afa7b40..32c467109722 100644 --- a/pkg/api/v1alpha1/cloudstackdatacenterconfig_types.go +++ b/pkg/api/v1alpha1/cloudstackdatacenterconfig_types.go @@ -15,6 +15,8 @@ package v1alpha1 import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -29,14 +31,15 @@ type CloudStackDatacenterConfigSpec struct { // Domain contains a grouping of accounts. Domains usually contain multiple accounts that have some logical relationship to each other and a set of delegated administrators with some authority over the domain and its subdomains // This field is considered as a fully qualified domain name which is the same as the domain path without "ROOT/" prefix. For example, if "foo" is specified then a domain with "ROOT/foo" domain path is picked. // The value "ROOT" is a special case that points to "the" ROOT domain of the CloudStack. That is, a domain with a path "ROOT/ROOT" is not allowed. - // - Domain string `json:"domain"` + Domain string `json:"domain,omitempty"` // Zones is a list of one or more zones that are managed by a single CloudStack management endpoint. - Zones []CloudStackZone `json:"zones"` + Zones []CloudStackZone `json:"zones,omitempty"` // Account typically represents a customer of the service provider or a department in a large organization. Multiple users can exist in an account, and all CloudStack resources belong to an account. Accounts have users and users have credentials to operate on resources within that account. If an account name is provided, a domain must also be provided. Account string `json:"account,omitempty"` // CloudStack Management API endpoint's IP. It is added to VM's noproxy list - ManagementApiEndpoint string `json:"managementApiEndpoint"` + ManagementApiEndpoint string `json:"managementApiEndpoint,omitempty"` + // AvailabilityZones list of different partitions to distribute VMs across - corresponds to a list of CAPI failure domains + AvailabilityZones []CloudStackAvailabilityZone `json:"availabilityZones,omitempty"` } type CloudStackResourceIdentifier struct { @@ -71,6 +74,24 @@ type CloudStackZone struct { Network CloudStackResourceIdentifier `json:"network"` } +// CloudStackAvailabilityZone maps to a CAPI failure domain to distribute machines across Cloudstack infrastructure +type CloudStackAvailabilityZone struct { + // Name is used as a unique identifier for each availability zone + Name string `json:"name"` + // CredentialRef is used to reference a secret in the eksa-system namespace + CredentialsRef string `json:"credentialsRef"` + // Zone represents the properties of the CloudStack zone in which clusters should be created, like the network. + Zone CloudStackZone `json:"zone"` + // Domain contains a grouping of accounts. Domains usually contain multiple accounts that have some logical relationship to each other and a set of delegated administrators with some authority over the domain and its subdomains + // This field is considered as a fully qualified domain name which is the same as the domain path without "ROOT/" prefix. For example, if "foo" is specified then a domain with "ROOT/foo" domain path is picked. + // The value "ROOT" is a special case that points to "the" ROOT domain of the CloudStack. That is, a domain with a path "ROOT/ROOT" is not allowed. + Domain string `json:"domain"` + // Account typically represents a customer of the service provider or a department in a large organization. Multiple users can exist in an account, and all CloudStack resources belong to an account. Accounts have users and users have credentials to operate on resources within that account. If an account name is provided, a domain must also be provided. + Account string `json:"account,omitempty"` + // CloudStack Management API endpoint's IP. It is added to VM's noproxy list + ManagementApiEndpoint string `json:"managementApiEndpoint"` +} + // CloudStackDatacenterConfigStatus defines the observed state of CloudStackDatacenterConfig type CloudStackDatacenterConfigStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file @@ -139,9 +160,34 @@ func (v *CloudStackDatacenterConfig) Marshallable() Marshallable { } func (v *CloudStackDatacenterConfig) Validate() error { + // TODO https://github.com/aws/eks-anywhere/issues/2406: Add validation to + // * Make sure that v.Spec.Zones, v.Spec.Domain, v.Spec.Account, v.Spec.ManagementApiEndpoint are all empty + // * Make sure that len(v.Spec.AvailabilityZones) > 0 + // * Make sure that all the AZ names are unique return nil } +func (v *CloudStackDatacenterConfig) SetDefaults() { + if v.Spec.AvailabilityZones == nil || len(v.Spec.AvailabilityZones) == 0 { + v.Spec.AvailabilityZones = make([]CloudStackAvailabilityZone, 0, len(v.Spec.Zones)) + for index, csZone := range v.Spec.Zones { + az := CloudStackAvailabilityZone{ + Name: fmt.Sprintf("availability-zone-%d", index), + Zone: csZone, + Account: v.Spec.Account, + Domain: v.Spec.Domain, + ManagementApiEndpoint: v.Spec.ManagementApiEndpoint, + CredentialsRef: "Global", + } + v.Spec.AvailabilityZones = append(v.Spec.AvailabilityZones, az) + } + } + v.Spec.Zones = nil + v.Spec.Domain = "" + v.Spec.Account = "" + v.Spec.ManagementApiEndpoint = "" +} + func (s *CloudStackDatacenterConfigSpec) Equal(o *CloudStackDatacenterConfigSpec) bool { if s == o { return true @@ -157,6 +203,19 @@ func (s *CloudStackDatacenterConfigSpec) Equal(o *CloudStackDatacenterConfigSpec return false } } + if len(s.AvailabilityZones) != len(o.AvailabilityZones) { + return false + } + oAzsMap := map[string]CloudStackAvailabilityZone{} + for _, oAz := range o.AvailabilityZones { + oAzsMap[oAz.Name] = oAz + } + for _, sAz := range s.AvailabilityZones { + oAz, found := oAzsMap[sAz.Name] + if !found || !sAz.Equal(&oAz) { + return false + } + } return s.ManagementApiEndpoint == o.ManagementApiEndpoint && s.Domain == o.Domain && s.Account == o.Account @@ -178,6 +237,21 @@ func (z *CloudStackZone) Equal(o *CloudStackZone) bool { return false } +func (az *CloudStackAvailabilityZone) Equal(o *CloudStackAvailabilityZone) bool { + if az == o { + return true + } + if az == nil || o == nil { + return false + } + return az.Zone.Equal(&o.Zone) && + az.Name == o.Name && + az.CredentialsRef == o.CredentialsRef && + az.Account == o.Account && + az.Domain == o.Domain && + az.ManagementApiEndpoint == o.ManagementApiEndpoint +} + // +kubebuilder:object:generate=false // Same as CloudStackDatacenterConfig except stripped down for generation of yaml file during generate clusterconfig diff --git a/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go b/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go index 9e7676c51c7b..2afae011c22e 100644 --- a/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go @@ -19,8 +19,6 @@ package tinkerbell -import () - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Action) DeepCopyInto(out *Action) { *out = *in diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 5b4609490aca..444a8ada5572 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -277,6 +277,22 @@ func (in *CiliumConfig) DeepCopy() *CiliumConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudStackAvailabilityZone) DeepCopyInto(out *CloudStackAvailabilityZone) { + *out = *in + out.Zone = in.Zone +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudStackAvailabilityZone. +func (in *CloudStackAvailabilityZone) DeepCopy() *CloudStackAvailabilityZone { + if in == nil { + return nil + } + out := new(CloudStackAvailabilityZone) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloudStackDatacenterConfig) DeepCopyInto(out *CloudStackDatacenterConfig) { *out = *in @@ -344,6 +360,11 @@ func (in *CloudStackDatacenterConfigSpec) DeepCopyInto(out *CloudStackDatacenter *out = make([]CloudStackZone, len(*in)) copy(*out, *in) } + if in.AvailabilityZones != nil { + in, out := &in.AvailabilityZones, &out.AvailabilityZones + *out = make([]CloudStackAvailabilityZone, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudStackDatacenterConfigSpec.