From 95a1ac946ee73e2b535dcf4177414776fd140872 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 9 Jan 2024 13:13:43 +0100 Subject: [PATCH 001/115] Add get storage profile method by ID Signed-off-by: abarreiro --- govcd/system.go | 15 +++++++++++++++ govcd/system_test.go | 9 +++++++-- types/v56/types.go | 1 + 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/govcd/system.go b/govcd/system.go index 29260e8e0..1047096bc 100644 --- a/govcd/system.go +++ b/govcd/system.go @@ -721,6 +721,21 @@ func getExtension(client *Client) (*types.Extension, error) { return extensions, err } +// GetStorageProfileById fetches a storage profile using its ID. +func (vcdClient *VCDClient) GetStorageProfileById(id string) (*types.VdcStorageProfile, error) { + storageProfileHref := vcdClient.Client.VCDHREF + storageProfileHref.Path += "/admin/vdcStorageProfile/" + extractUuid(id) + + vdcStorageProfile := &types.VdcStorageProfile{} + + _, err := vcdClient.Client.ExecuteRequest(storageProfileHref.String(), http.MethodGet, "", "error retrieving storage profile: %s", nil, vdcStorageProfile) + if err != nil { + return nil, err + } + + return vdcStorageProfile, nil +} + // GetStorageProfileByHref fetches storage profile using provided HREF. // Deprecated: use client.GetStorageProfileByHref or vcdClient.GetStorageProfileByHref func GetStorageProfileByHref(vcdClient *VCDClient, url string) (*types.VdcStorageProfile, error) { diff --git a/govcd/system_test.go b/govcd/system_test.go index a12816ee3..acf237da9 100644 --- a/govcd/system_test.go +++ b/govcd/system_test.go @@ -676,8 +676,8 @@ func (vcd *TestVCD) Test_QueryNetworkPoolByName(check *C) { } -// Test getting storage profile by href and vdc client -func (vcd *TestVCD) Test_GetStorageProfileByHref(check *C) { +// Test_GetStorageProfile tests all the getters of Storage Profile +func (vcd *TestVCD) Test_GetStorageProfile(check *C) { if vcd.config.VCD.ProviderVdc.StorageProfile == "" { check.Skip("Skipping test because storage profile is not configured") } @@ -699,6 +699,11 @@ func (vcd *TestVCD) Test_GetStorageProfileByHref(check *C) { check.Assert(foundStorageProfile.IopsSettings, NotNil) check.Assert(foundStorageProfile, Not(Equals), types.VdcStorageProfile{}) check.Assert(foundStorageProfile.IopsSettings, Not(Equals), types.VdcStorageProfileIopsSettings{}) + + // Get storage profile by ID + foundStorageProfile2, err := vcd.client.GetStorageProfileById(foundStorageProfile.ID) + check.Assert(err, IsNil) + check.Assert(foundStorageProfile, DeepEquals, foundStorageProfile2) } func (vcd *TestVCD) Test_GetOrgList(check *C) { diff --git a/types/v56/types.go b/types/v56/types.go index dc83fb091..8bd6292ef 100644 --- a/types/v56/types.go +++ b/types/v56/types.go @@ -583,6 +583,7 @@ type VdcStorageProfileConfiguration struct { // https://vdc-repo.vmware.com/vmwb-repository/dcr-public/7a028e78-bd37-4a6a-8298-9c26c7eeb9aa/09142237-dd46-4dee-8326-e07212fb63a8/doc/doc/types/VdcStorageProfileType.html // https://vdc-repo.vmware.com/vmwb-repository/dcr-public/71e12563-bc11-4d64-821d-92d30f8fcfa1/7424bf8e-aec2-44ad-be7d-b98feda7bae0/doc/doc/types/AdminVdcStorageProfileType.html type VdcStorageProfile struct { + ID string `xml:"id,attr"` Xmlns string `xml:"xmlns,attr"` Name string `xml:"name,attr"` Enabled *bool `xml:"Enabled,omitempty"` From 55999bff622f4f3090dba13f122bd14626288ef1 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 26 Jan 2024 14:10:52 +0100 Subject: [PATCH 002/115] Add CSE methods Signed-off-by: abarreiro --- govcd/cse.go | 188 +++++++++++++ govcd/cse/4.2/capiyaml_cluster.tmpl | 153 +++++++++++ govcd/cse/4.2/capiyaml_mhc.tmpl | 22 ++ govcd/cse/4.2/capiyaml_nodepool.tmpl | 41 +++ govcd/cse/4.2/rde.tmpl | 31 +++ govcd/cse/tkg_versions.json | 92 +++++++ govcd/cse_test.go | 20 ++ govcd/cse_util.go | 389 +++++++++++++++++++++++++++ types/v56/cse.go | 183 +++++++++++++ 9 files changed, 1119 insertions(+) create mode 100644 govcd/cse.go create mode 100644 govcd/cse/4.2/capiyaml_cluster.tmpl create mode 100644 govcd/cse/4.2/capiyaml_mhc.tmpl create mode 100644 govcd/cse/4.2/capiyaml_nodepool.tmpl create mode 100644 govcd/cse/4.2/rde.tmpl create mode 100644 govcd/cse/tkg_versions.json create mode 100644 govcd/cse_test.go create mode 100644 govcd/cse_util.go create mode 100644 types/v56/cse.go diff --git a/govcd/cse.go b/govcd/cse.go new file mode 100644 index 000000000..c8b0a9ba2 --- /dev/null +++ b/govcd/cse.go @@ -0,0 +1,188 @@ +package govcd + +import ( + _ "embed" + "encoding/json" + "fmt" + "github.com/vmware/go-vcloud-director/v2/types/v56" + "github.com/vmware/go-vcloud-director/v2/util" + "strings" + "time" +) + +// supportedCseVersions is a map that contains only the supported CSE versions as keys, +// and its corresponding components versions as a slice of strings. The first string is the VCDKEConfig RDE Type version, +// then the CAPVCD RDE Type version and finally the CAPVCD Behavior version. +// TODO: Is this really necessary? What happens in UI if I have a 1.1.0-1.2.0-1.0.0 (4.2) cluster and then CSE is updated to 4.3? +var supportedCseVersions = map[string][]string{ + "4.2": { + "1.1.0", // VCDKEConfig RDE Type version + "1.2.0", // CAPVCD RDE Type version + "1.0.0", // CAPVCD Behavior version + }, +} + +// CseClusterApiProviderCluster is a type for handling ClusterApiProviderVCD (CAPVCD) cluster instances created +// by the Container Service Extension (CSE) +type CseClusterApiProviderCluster struct { + Capvcd *types.Capvcd + ID string + Etag string + client *Client +} + +// CseClusterCreationInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// must set in order to create a Kubernetes cluster. +type CseClusterCreationInput struct { + Name string + OrganizationId string + VdcId string + NetworkId string + KubernetesTemplateOvaId string + CseVersion string + ControlPlane ControlPlaneInput + WorkerPools []WorkerPoolInput + DefaultStorageClass *DefaultStorageClassInput // Optional + Owner string // Optional, if not set will pick the current user present in the VCDClient + ApiToken string + NodeHealthCheck bool +} + +// ControlPlaneInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// must set in order to specify the Control Plane inside a CseClusterCreationInput object. +type ControlPlaneInput struct { + MachineCount int + DiskSizeGi int + SizingPolicyId string // Optional + PlacementPolicyId string // Optional + StorageProfileId string // Optional + Ip string // Optional +} + +// WorkerPoolInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// must set in order to specify one Worker Pool inside a CseClusterCreationInput object. +type WorkerPoolInput struct { + Name string + MachineCount int + DiskSizeGi int + SizingPolicyId string // Optional + PlacementPolicyId string // Optional + VGpuPolicyId string // Optional + StorageProfileId string // Optional +} + +// DefaultStorageClassInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// must set in order to specify a Default Storage Class inside a CseClusterCreationInput object. +type DefaultStorageClassInput struct { + StorageProfileId string + Name string + ReclaimPolicy string + Filesystem string +} + +//go:embed cse/tkg_versions.json +var cseTkgVersionsJson []byte + +// CseCreateKubernetesCluster creates a Kubernetes cluster with the data given as input (CseClusterCreationInput). If the given +// timeout is 0, it waits forever for the cluster creation. Otherwise, if the timeout is reached and the cluster is not available, +// it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster in the returned CseClusterApiProviderCluster. +// If the cluster is created correctly, returns all the data in CseClusterApiProviderCluster. +func (vcdClient *VCDClient) CseCreateKubernetesCluster(clusterData CseClusterCreationInput, timeoutMinutes time.Duration) (*CseClusterApiProviderCluster, error) { + _, err := getRemoteFile(&vcdClient.Client, fmt.Sprintf("%s/capiyaml_cluster.tmpl", clusterData.CseVersion)) + if err != nil { + return nil, err + } + + _, err = clusterData.toCseClusterCreationPayload(vcdClient) + if err != nil { + return nil, err + } + + err = waitUntilClusterIsProvisioned(vcdClient, "", timeoutMinutes) + if err != nil { + return nil, err + } + + result := &CseClusterApiProviderCluster{ + client: &vcdClient.Client, + ID: "", + } + return result, nil +} + +// CseConvertToCapvcdCluster takes the receiver, which is a generic RDE that must represent an existing CSE Kubernetes cluster, +// and transforms it to a specific Container Service Extension CAPVCD object that represents the same cluster, but +// it is easy to explore and consume. If the receiver object does not contain a CAPVCD object, this method +// will obviously return an error. +func (rde *DefinedEntity) CseConvertToCapvcdCluster() (*CseClusterApiProviderCluster, error) { + requiredType := "vmware:capvcdCluster" + + if !strings.Contains(rde.DefinedEntity.ID, requiredType) || !strings.Contains(rde.DefinedEntity.EntityType, requiredType) { + return nil, fmt.Errorf("the receiver RDE is not a '%s' entity, it is '%s'", requiredType, rde.DefinedEntity.EntityType) + } + + entityBytes, err := json.Marshal(rde.DefinedEntity.Entity) + if err != nil { + return nil, fmt.Errorf("could not marshal the RDE contents to create a Capvcd instance: %s", err) + } + + result := &CseClusterApiProviderCluster{ + Capvcd: &types.Capvcd{}, + Etag: rde.Etag, + client: rde.client, + } + + err = json.Unmarshal(entityBytes, result.Capvcd) + if err != nil { + return nil, fmt.Errorf("could not unmarshal the RDE contents to create a Capvcd instance: %s", err) + } + return result, nil +} + +// waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if "operations_timeout_minutes=0") +// or until this timeout is reached. If one of the states is "error", this function also checks whether "auto_repair_on_errors=true" to keep +// waiting. +func waitUntilClusterIsProvisioned(vcdClient *VCDClient, clusterId string, timeoutMinutes time.Duration) error { + var elapsed time.Duration + logHttpResponse := util.LogHttpResponse + sleepTime := 30 + + // The following loop is constantly polling VCD to retrieve the RDE, which has a big JSON inside, so we avoid filling + // the log with these big payloads. We use defer to be sure that we restore the initial logging state. + defer func() { + util.LogHttpResponse = logHttpResponse + }() + + start := time.Now() + latestState := "" + for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies operations_timeout_minutes=0, we wait forever + util.LogHttpResponse = false + rde, err := vcdClient.GetRdeById(clusterId) + util.LogHttpResponse = logHttpResponse + if err != nil { + return err + } + + capvcdCluster, err := rde.CseConvertToCapvcdCluster() + if err != nil { + return err + } + + latestState = capvcdCluster.Capvcd.Status.VcdKe.State + switch latestState { + case "provisioned": + return nil + case "error": + // We just finish if auto-recovery is disabled, otherwise we just let CSE fixing things in background + if !capvcdCluster.Capvcd.Spec.VcdKe.AutoRepairOnErrors { + // Try to give feedback about what went wrong, which is located in a set of events in the RDE payload + return fmt.Errorf("got an error and 'auto_repair_on_errors=false', aborting. Errors: %s", capvcdCluster.Capvcd.Status.Capvcd.ErrorSet[len(capvcdCluster.Capvcd.Status.Capvcd.ErrorSet)-1].AdditionalDetails.DetailedError) + } + } + + util.Logger.Printf("[DEBUG] Cluster '%s' is in '%s' state, will check again in %d seconds", capvcdCluster.ID, latestState, sleepTime) + elapsed = time.Since(start) + time.Sleep(time.Duration(sleepTime) * time.Second) + } + return fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, latestState) +} diff --git a/govcd/cse/4.2/capiyaml_cluster.tmpl b/govcd/cse/4.2/capiyaml_cluster.tmpl new file mode 100644 index 000000000..16a676ae1 --- /dev/null +++ b/govcd/cse/4.2/capiyaml_cluster.tmpl @@ -0,0 +1,153 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" + labels: + cluster-role.tkg.tanzu.vmware.com/management: "" + tanzuKubernetesRelease: "{{.TkrVersion}}" + tkg.tanzu.vmware.com/cluster-name: "{{.ClusterName}}" + annotations: + osInfo: "ubuntu,20.04,amd64" + TKGVERSION: "{{.TkgVersion}}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - "{{.PodCidr}}" + serviceDomain: cluster.local + services: + cidrBlocks: + - "{{.ServiceCidr}}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "{{.ClusterName}}-control-plane-node-pool" + namespace: "{{.TargetNamespace}}" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: VCDCluster + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" +--- +apiVersion: v1 +kind: Secret +metadata: + name: capi-user-credentials + namespace: {{.TargetNamespace}} +type: Opaque +data: + username: "{{.UsernameB64}}" + refreshToken: "{{.ApiTokenB64}}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: VCDCluster +metadata: + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" +spec: + site: "{{.VcdSite}}" + org: "{{.Org}}" + ovdc: "{{.OrgVdc}}" + ovdcNetwork: "{{.OrgVdcNetwork}}" + {{- if .ControlPlaneEndpoint}} + controlPlaneEndpoint: + host: "{{.ControlPlaneEndpoint}}" + port: 6443 + {{- end}} + {{- if .VirtualIpSubnet}} + loadBalancerConfigSpec: + vipSubnet: "{{.VirtualIpSubnet}}" + {{- end}} + useAsManagementCluster: false + userContext: + secretRef: + name: capi-user-credentials + namespace: "{{.TargetNamespace}}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: VCDMachineTemplate +metadata: + name: "{{.ClusterName}}-control-plane-node-pool" + namespace: "{{.TargetNamespace}}" +spec: + template: + spec: + catalog: "{{.Catalog}}" + template: "{{.VAppTemplate}}" + sizingPolicy: "{{.ControlPlaneSizingPolicy}}" + placementPolicy: "{{.ControlPlanePlacementPolicy}}" + storageProfile: "{{.ControlPlaneStorageProfile}}" + diskSize: {{.ControlPlaneDiskSize}} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "{{.ClusterName}}-control-plane-node-pool" + namespace: "{{.TargetNamespace}}" +spec: + kubeadmConfigSpec: + preKubeadmCommands: + - mv /etc/ssl/certs/custom_certificate_*.crt /usr/local/share/ca-certificates && update-ca-certificates + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + dns: + imageRepository: "{{.ContainerRegistryUrl}}" + imageTag: "{{.DnsVersion}}" + etcd: + local: + imageRepository: "{{.ContainerRegistryUrl}}" + imageTag: "{{.EtcdVersion}}" + imageRepository: "{{.ContainerRegistryUrl}}" + users: + - name: root + sshAuthorizedKeys: + - "{{.SshPublicKey}}" + initConfiguration: + nodeRegistration: + criSocket: /run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cloud-provider: external + joinConfiguration: + nodeRegistration: + criSocket: /run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cloud-provider: external + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: VCDMachineTemplate + name: "{{.ClusterName}}-control-plane-node-pool" + namespace: "{{.TargetNamespace}}" + replicas: {{.ControlPlaneMachineCount}} + version: "{{.KubernetesVersion}}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "{{.ClusterName}}-kct" + namespace: "{{.TargetNamespace}}" +spec: + template: + spec: + users: + - name: root + sshAuthorizedKeys: + - "{{.SshPublicKey}}" + useExperimentalRetryJoin: true + preKubeadmCommands: + - mv /etc/ssl/certs/custom_certificate_*.crt /usr/local/share/ca-certificates && update-ca-certificates + joinConfiguration: + nodeRegistration: + criSocket: /run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cloud-provider: external \ No newline at end of file diff --git a/govcd/cse/4.2/capiyaml_mhc.tmpl b/govcd/cse/4.2/capiyaml_mhc.tmpl new file mode 100644 index 000000000..d31e4c3ec --- /dev/null +++ b/govcd/cse/4.2/capiyaml_mhc.tmpl @@ -0,0 +1,22 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" + labels: + clusterctl.cluster.x-k8s.io: "" + clusterctl.cluster.x-k8s.io/move: "" +spec: + clusterName: "{{.ClusterName}}" + maxUnhealthy: "{{.MaxUnhealthyNodePercentage}}" + nodeStartupTimeout: "{{.NodeStartupTimeout}}" + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: "{{.ClusterName}}" + unhealthyConditions: + - type: Ready + status: Unknown + timeout: "{{.NodeUnknownTimeout}}" + - type: Ready + status: "False" + timeout: "{{.NodeNotReadyTimeout}}" \ No newline at end of file diff --git a/govcd/cse/4.2/capiyaml_nodepool.tmpl b/govcd/cse/4.2/capiyaml_nodepool.tmpl new file mode 100644 index 000000000..e2292c7d7 --- /dev/null +++ b/govcd/cse/4.2/capiyaml_nodepool.tmpl @@ -0,0 +1,41 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: VCDMachineTemplate +metadata: + name: "{{.NodePoolName}}" + namespace: "{{.TargetNamespace}}" +spec: + template: + spec: + catalog: "{{.Catalog}}" + template: "{{.VAppTemplate}}" + sizingPolicy: "{{.NodePoolSizingPolicy}}" + placementPolicy: "{{.NodePoolPlacementPolicy}}" + storageProfile: "{{.NodePoolStorageProfile}}" + diskSize: "{{.NodePoolDiskSize}}" + enableNvidiaGPU: {{.NodePoolEnableGpu}} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "{{.NodePoolName}}" + namespace: "{{.TargetNamespace}}" +spec: + clusterName: "{{.ClusterName}}" + replicas: {{.NodePoolMachineCount}} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: "{{.ClusterName}}-kct" + namespace: "{{.TargetNamespace}}" + clusterName: "{{.ClusterName}}" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: VCDMachineTemplate + name: "{{.NodePoolName}}" + namespace: "{{.TargetNamespace}}" + version: "{{.KubernetesVersion}}" \ No newline at end of file diff --git a/govcd/cse/4.2/rde.tmpl b/govcd/cse/4.2/rde.tmpl new file mode 100644 index 000000000..e5ea3e2b8 --- /dev/null +++ b/govcd/cse/4.2/rde.tmpl @@ -0,0 +1,31 @@ +{ + "apiVersion": "capvcd.vmware.com/v1.1", + "kind": "CAPVCDCluster", + "name": "{{.Name}}", + "metadata": { + "name": "{{.Name}}", + "orgName": "{{.Org}}", + "site": "{{.VcdUrl}}", + "virtualDataCenterName": "{{.Vdc}}" + }, + "spec": { + "vcdKe": { + "isVCDKECluster": true, + "markForDelete": {{.Delete}}, + "forceDelete": {{.ForceDelete}}, + "autoRepairOnErrors": {{.AutoRepairOnErrors}}, + {{- if .DefaultStorageClassName }} + "defaultStorageClassOptions": { + "filesystem": "{{.DefaultStorageClassFileSystem}}", + "k8sStorageClassName": "{{.DefaultStorageClassName}}", + "vcdStorageProfileName": "{{.DefaultStorageClassStorageProfile}}", + "useDeleteReclaimPolicy": {{.DefaultStorageClassUseDeleteReclaimPolicy}} + }, + {{- end }} + "secure": { + "apiToken": "{{.ApiToken}}" + } + }, + "capiYaml": "{{.CapiYaml}}" + } +} diff --git a/govcd/cse/tkg_versions.json b/govcd/cse/tkg_versions.json new file mode 100644 index 000000000..0566126b9 --- /dev/null +++ b/govcd/cse/tkg_versions.json @@ -0,0 +1,92 @@ +{ + "v1.27.5+vmware.1-tkg.1-0eb96d2f9f4f705ac87c40633d4b69st": { + "tkg": "v2.4.0", + "etcd": "v3.5.7_vmware.6", + "coreDns": "v1.10.1_vmware.7" + }, + "v1.26.8+vmware.1-tkg.1-b8c57a6c8c98d227f74e7b1a9eef27st": { + "tkg": "v2.4.0", + "etcd": "v3.5.6_vmware.20", + "coreDns": "v1.10.1_vmware.7" + }, + "v1.26.8+vmware.1-tkg.1-0edd4dafbefbdb503f64d5472e500cf8": { + "tkg": "v2.3.1", + "etcd": "v3.5.6_vmware.20", + "coreDns": "v1.9.3_vmware.16" + }, + "v1.25.13+vmware.1-tkg.1-0031669997707d1c644156b8fc31ebst": { + "tkg": "v2.4.0", + "etcd": "v3.5.6_vmware.20", + "coreDns": "v1.10.1_vmware.7" + }, + "v1.25.13+vmware.1-tkg.1-6f7650434fd3787d751e8fb3c9e2153d": { + "tkg": "v2.3.1", + "etcd": "v3.5.6_vmware.20", + "coreDns": "v1.9.3_vmware.11" + }, + "v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc": { + "tkg": "v2.2.0", + "etcd": "v3.5.6_vmware.9", + "coreDns": "v1.9.3_vmware.8" + }, + "v1.24.17+vmware.1-tkg.1-9f70d901a7d851fb115411e6790fdeae": { + "tkg": "v2.3.1", + "etcd": "v3.5.6_vmware.19", + "coreDns": "v1.8.6_vmware.26" + }, + "v1.24.11+vmware.1-tkg.1-2ccb2a001f8bd8f15f1bfbc811071830": { + "tkg": "v2.2.0", + "etcd": "v3.5.6_vmware.10", + "coreDns": "v1.8.6_vmware.18" + }, + "v1.24.10+vmware.1-tkg.1-765d418b72c247c2310384e640ee075e": { + "tkg": "v2.1.1", + "etcd": "v3.5.6_vmware.6", + "coreDns": "v1.8.6_vmware.17" + }, + "v1.23.17+vmware.1-tkg.1-ee4d95d5d08cd7f31da47d1480571754": { + "tkg": "v2.2.0", + "etcd": "v3.5.6_vmware.11", + "coreDns": "v1.8.6_vmware.19" + }, + "v1.23.16+vmware.1-tkg.1-eb0de9755338b944ea9652e6f758b3ce": { + "tkg": "v2.1.1", + "etcd": "v3.5.6_vmware.5", + "coreDns": "v1.8.6_vmware.16" + }, + "v1.22.17+vmware.1-tkg.1-df08b304658a6cf17f5e74dc0ab7543c": { + "tkg": "v2.1.1", + "etcd": "v3.5.6_vmware.1", + "coreDns": "v1.8.4_vmware.10" + }, + "v1.22.9+vmware.1-tkg.1-2182cbabee08edf480ee9bc5866d6933": { + "tkg": "v1.5.4", + "etcd": "v3.5.4_vmware.2", + "coreDns": "v1.8.4_vmware.9" + }, + "v1.21.11+vmware.1-tkg.2-d788dbbb335710c0a0d1a28670057896": { + "tkg": "v1.5.4", + "etcd": "v3.4.13_vmware.27", + "coreDns": "v1.8.0_vmware.13" + }, + "v1.21.8+vmware.1-tkg.2-ed3c93616a02968be452fe1934a1d37c": { + "tkg": "v1.4.3", + "etcd": "v3.4.13_vmware.25", + "coreDns": "v1.8.0_vmware.11" + }, + "v1.20.15+vmware.1-tkg.2-839faf7d1fa7fa356be22b72170ce1a8": { + "tkg": "v1.5.4", + "etcd": "v3.4.13_vmware.23", + "coreDns": "v1.7.0_vmware.15" + }, + "v1.20.14+vmware.1-tkg.2-5a5027ce2528a6229acb35b38ff8084e": { + "tkg": "v1.4.3", + "etcd": "v3.4.13_vmware.23", + "coreDns": "v1.7.0_vmware.15" + }, + "v1.19.16+vmware.1-tkg.2-fba68db15591c15fcd5f26b512663a42": { + "tkg": "v1.4.3", + "etcd": "v3.4.13_vmware.19", + "coreDns": "v1.7.0_vmware.15" + } +} diff --git a/govcd/cse_test.go b/govcd/cse_test.go new file mode 100644 index 000000000..a7a126376 --- /dev/null +++ b/govcd/cse_test.go @@ -0,0 +1,20 @@ +//go:build functional || openapi || cse || ALL + +/* + * Copyright 2024 VMware, Inc. All rights reserved. Licensed under the Apache v2 License. + */ + +package govcd + +import ( + "fmt" + . "gopkg.in/check.v1" +) + +// Test_Cse +func (vcd *TestVCD) Test_Cse(check *C) { + if vcd.skipAdminTests { + check.Skip(fmt.Sprintf(TestRequiresSysAdminPrivileges, check.TestName())) + } + +} diff --git a/govcd/cse_util.go b/govcd/cse_util.go new file mode 100644 index 000000000..a63667ec5 --- /dev/null +++ b/govcd/cse_util.go @@ -0,0 +1,389 @@ +package govcd + +import ( + _ "embed" + "encoding/json" + "fmt" + "github.com/vmware/go-vcloud-director/v2/types/v56" + "github.com/vmware/go-vcloud-director/v2/util" + "io" + "net/http" + "regexp" + "strings" +) + +// cseClusterCreationGoTemplateArguments defines the required arguments that are required by the Go templates used internally to specify +// a Kubernetes cluster. These are not set by the user, but instead they are computed from a valid +// CseClusterCreationInput object in the CseClusterCreationInput.toCseClusterCreationPayload method. These fields are then +// inserted in Go templates to render a final JSON that is valid to be used as the cluster Runtime Defined Entity (RDE) payload. +type cseClusterCreationGoTemplateArguments struct { + Name string + OrganizationName string + VdcName string + NetworkName string + KubernetesTemplateOvaName string + TkgVersionBundle tkgVersionBundle + CatalogName string + RdeType *types.DefinedEntityType + ControlPlane controlPlane + WorkerPools []workerPool + DefaultStorageClass defaultStorageClass + VcdKeConfig *vcdKeConfig + Owner string + ApiToken string + VcdUrl string +} + +// controlPlane defines the Control Plane inside cseClusterCreationGoTemplateArguments +type controlPlane struct { + MachineCount int + DiskSizeGi int + SizingPolicyName string + PlacementPolicyName string + StorageProfileName string + Ip string +} + +// workerPool defines a Worker pool inside cseClusterCreationGoTemplateArguments +type workerPool struct { + Name string + MachineCount int + DiskSizeGi int + SizingPolicyName string + PlacementPolicyName string + VGpuPolicyName string + StorageProfileName string +} + +// defaultStorageClass defines a Default Storage Class inside cseClusterCreationGoTemplateArguments +type defaultStorageClass struct { + StorageProfileName string + Name string + ReclaimPolicy string + Filesystem string +} + +// vcdKeConfig is a type that contains only the required and relevant fields from the Container Service Extension (CSE) installation configuration, +// such as the Machine Health Check settings or the container registry URL. +type vcdKeConfig struct { + MaxUnhealthyNodesPercentage float64 + NodeStartupTimeout string + NodeNotReadyTimeout string + NodeUnknownTimeout string + ContainerRegistryUrl string +} + +// validate validates the CSE Kubernetes cluster creation input data. Returns an error if some of the fields is wrong. +func (ccd *CseClusterCreationInput) validate() error { + cseNamesRegex, err := regexp.Compile(`^[a-z](?:[a-z0-9-]{0,29}[a-z0-9])?$`) + if err != nil { + return fmt.Errorf("could not compile regular expression '%s'", err) + } + + if !cseNamesRegex.MatchString(ccd.Name) { + return fmt.Errorf("the cluster name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", ccd.Name) + } + + if ccd.OrganizationId == "" { + return fmt.Errorf("the Organization ID is required") + } + if ccd.VdcId == "" { + return fmt.Errorf("the VDC ID is required") + } + if ccd.KubernetesTemplateOvaId == "" { + return fmt.Errorf("the Kubernetes template OVA ID is required") + } + if ccd.NetworkId == "" { + return fmt.Errorf("the Network ID is required") + } + if _, ok := supportedCseVersions[ccd.CseVersion]; !ok { + return fmt.Errorf("the CSE version '%s' is not supported. Must be one of %v", ccd.CseVersion, getKeys(supportedCseVersions)) + } + if ccd.ControlPlane.MachineCount < 1 || ccd.ControlPlane.MachineCount%2 == 0 { + return fmt.Errorf("number of control plane nodes must be odd and higher than 0, but it was '%d'", ccd.ControlPlane.MachineCount) + } + if ccd.ControlPlane.DiskSizeGi < 20 { + return fmt.Errorf("disk size for the Control Plane in Gibibytes (Gi) must be at least 20, but it was '%d'", ccd.ControlPlane.DiskSizeGi) + } + if len(ccd.WorkerPools) == 0 { + return fmt.Errorf("there must be at least one Worker pool") + } + for _, workerPool := range ccd.WorkerPools { + if !cseNamesRegex.MatchString(workerPool.Name) { + return fmt.Errorf("the Worker pool name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", workerPool.Name) + } + if workerPool.DiskSizeGi < 20 { + return fmt.Errorf("disk size for the Worker pool '%s' in Gibibytes (Gi) must be at least 20, but it was '%d'", workerPool.Name, workerPool.DiskSizeGi) + } + if workerPool.MachineCount < 1 { + return fmt.Errorf("number of Worker pool '%s' nodes must higher than 0, but it was '%d'", workerPool.Name, workerPool.MachineCount) + } + } + if ccd.DefaultStorageClass != nil { + if !cseNamesRegex.MatchString(ccd.DefaultStorageClass.Name) { + return fmt.Errorf("the Default Storage Class name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", ccd.DefaultStorageClass.Name) + } + if ccd.DefaultStorageClass.StorageProfileId == "" { + return fmt.Errorf("the Storage Profile ID for the Default Storage Class is required") + } + if ccd.DefaultStorageClass.ReclaimPolicy != "delete" && ccd.DefaultStorageClass.ReclaimPolicy != "retain" { + return fmt.Errorf("the reclaim policy for the Default Storage Class must be either 'delete' or 'retain', but it was '%s'", ccd.DefaultStorageClass.ReclaimPolicy) + } + if ccd.DefaultStorageClass.Filesystem != "ext4" && ccd.DefaultStorageClass.ReclaimPolicy != "xfs" { + return fmt.Errorf("the filesystem for the Default Storage Class must be either 'ext4' or 'xfs', but it was '%s'", ccd.DefaultStorageClass.Filesystem) + } + } + if ccd.ApiToken == "" { + return fmt.Errorf("the API token is required") + } + + return nil +} + +// toCseClusterCreationPayload transforms user input data (receiver CseClusterCreationInput) into the final payload that +// will be used to render the Go templates that define a Kubernetes cluster creation payload (cseClusterCreationGoTemplateArguments). +func (input *CseClusterCreationInput) toCseClusterCreationPayload(vcdClient *VCDClient) (*cseClusterCreationGoTemplateArguments, error) { + output := &cseClusterCreationGoTemplateArguments{} + err := input.validate() + if err != nil { + return nil, err + } + org, err := vcdClient.GetOrgById(input.OrganizationId) + if err != nil { + return nil, fmt.Errorf("could not retrieve the Organization with ID '%s': %s", input.VdcId, err) + } + output.OrganizationName = org.Org.Name + + vdc, err := org.GetVDCById(input.VdcId, true) + if err != nil { + return nil, fmt.Errorf("could not retrieve the VDC with ID '%s': %s", input.VdcId, err) + } + output.VdcName = vdc.Vdc.Name + + vAppTemplate, err := getVAppTemplateById(org.client, input.KubernetesTemplateOvaId) + if err != nil { + return nil, fmt.Errorf("could not retrieve the Kubernetes OVA with ID '%s': %s", input.KubernetesTemplateOvaId, err) + } + output.KubernetesTemplateOvaName = vAppTemplate.VAppTemplate.Name + + tkgVersions, err := getTkgVersionBundleFromVAppTemplateName(vAppTemplate.VAppTemplate.Name) + if err != nil { + return nil, err + } + output.TkgVersionBundle = tkgVersions + + catalogName, err := vAppTemplate.GetCatalogName() + if err != nil { + return nil, fmt.Errorf("could not retrieve the Catalog name of the OVA '%s': %s", input.KubernetesTemplateOvaId, err) + } + output.CatalogName = catalogName + + network, err := vdc.GetOrgVdcNetworkById(input.NetworkId, true) + if err != nil { + return nil, fmt.Errorf("could not retrieve the Org VDC Network with ID '%s': %s", input.NetworkId, err) + } + output.NetworkName = network.OrgVDCNetwork.Name + + currentCseVersion := supportedCseVersions[input.CseVersion] + rdeType, err := vcdClient.GetRdeType("vmware", "capvcdCluster", currentCseVersion[1]) + if err != nil { + return nil, fmt.Errorf("could not retrieve RDE Type vmware:capvcdCluster:'%s': %s", currentCseVersion[1], err) + } + output.RdeType = rdeType.DefinedEntityType + + // The input to create a cluster uses different entities IDs, but CSE cluster creation process uses names. + // For that reason, we need to transform IDs to Names by querying VCD. This process is optimized with a tiny "cache" map. + idToNameCache := map[string]string{ + "": "", // Default empty value to map optional values that were not set + } + var computePolicyIds []string + var storageProfileIds []string + for _, w := range input.WorkerPools { + computePolicyIds = append(computePolicyIds, w.SizingPolicyId, w.PlacementPolicyId, w.VGpuPolicyId) + storageProfileIds = append(storageProfileIds, w.StorageProfileId) + } + computePolicyIds = append(computePolicyIds, input.ControlPlane.SizingPolicyId, input.ControlPlane.PlacementPolicyId) + storageProfileIds = append(storageProfileIds, input.ControlPlane.StorageProfileId, input.DefaultStorageClass.StorageProfileId) + + for _, id := range storageProfileIds { + if _, alreadyPresent := idToNameCache[id]; !alreadyPresent { + storageProfile, err := vcdClient.GetStorageProfileById(id) + if err != nil { + return nil, fmt.Errorf("could not get Storage Profile with ID '%s': %s", id, err) + } + idToNameCache[id] = storageProfile.Name + } + } + for _, id := range computePolicyIds { + if _, alreadyPresent := idToNameCache[id]; !alreadyPresent { + computePolicy, err := vcdClient.GetVdcComputePolicyV2ById(id) + if err != nil { + return nil, fmt.Errorf("could not get Compute Policy with ID '%s': %s", id, err) + } + idToNameCache[id] = computePolicy.VdcComputePolicyV2.Name + } + } + + // Now that everything is cached in memory, we can build the Node pools and Storage Class payloads + output.WorkerPools = make([]workerPool, len(input.WorkerPools)) + for i, w := range input.WorkerPools { + output.WorkerPools[i] = workerPool{ + Name: w.Name, + MachineCount: w.MachineCount, + DiskSizeGi: w.DiskSizeGi, + } + output.WorkerPools[i].SizingPolicyName = idToNameCache[w.SizingPolicyId] + output.WorkerPools[i].PlacementPolicyName = idToNameCache[w.PlacementPolicyId] + output.WorkerPools[i].VGpuPolicyName = idToNameCache[w.VGpuPolicyId] + output.WorkerPools[i].StorageProfileName = idToNameCache[w.StorageProfileId] + } + output.ControlPlane = controlPlane{ + MachineCount: input.ControlPlane.MachineCount, + DiskSizeGi: input.ControlPlane.DiskSizeGi, + SizingPolicyName: idToNameCache[input.ControlPlane.SizingPolicyId], + PlacementPolicyName: idToNameCache[input.ControlPlane.PlacementPolicyId], + StorageProfileName: idToNameCache[input.ControlPlane.StorageProfileId], + } + + if input.DefaultStorageClass != nil { + output.DefaultStorageClass = defaultStorageClass{ + StorageProfileName: idToNameCache[input.DefaultStorageClass.StorageProfileId], + Name: input.DefaultStorageClass.Name, + ReclaimPolicy: input.DefaultStorageClass.ReclaimPolicy, + Filesystem: input.DefaultStorageClass.Filesystem, + } + } + + vcdKeConfig, err := getVcdKeConfiguration(vcdClient, input.CseVersion, input.NodeHealthCheck) + if err != nil { + return nil, err + } + output.VcdKeConfig = vcdKeConfig + + output.Owner = input.Owner + if input.Owner == "" { + sessionInfo, err := vcdClient.Client.GetSessionInfo() + if err != nil { + return nil, fmt.Errorf("error getting the owner of the cluster: %s", err) + } + output.Owner = sessionInfo.User.Name + } + output.VcdUrl = strings.Replace(vcdClient.Client.VCDHREF.String(), "/api", "", 1) + + return output, nil +} + +// tkgVersionBundle is a type that contains all the versions of the components of +// a Kubernetes cluster that can be obtained with the vApp Template name, downloaded +// from VMware Customer connect: +// https://customerconnect.vmware.com/downloads/details?downloadGroup=TKG-240&productId=1400 +type tkgVersionBundle struct { + EtcdVersion string + CoreDnsVersion string + TkgVersion string + TkrVersion string + KubernetesVersion string +} + +// getTkgVersionBundleFromVAppTemplateName returns a tkgVersionBundle with the details of +// all the Kubernetes cluster components versions given a valid vApp Template name, that should +// correspond to a Kubernetes template. If it is not a valid vApp Template, returns an error. +func getTkgVersionBundleFromVAppTemplateName(ovaName string) (tkgVersionBundle, error) { + result := tkgVersionBundle{} + + if strings.Contains(ovaName, "photon") { + return result, fmt.Errorf("the OVA '%s' uses Photon, and it is not supported", ovaName) + } + + cutPosition := strings.LastIndex(ovaName, "kube-") + if cutPosition < 0 { + return result, fmt.Errorf("the OVA '%s' is not a Kubernetes template OVA", ovaName) + } + parsedOvaName := strings.ReplaceAll(ovaName, ".ova", "")[cutPosition+len("kube-"):] + + versionsMap := map[string]interface{}{} + err := json.Unmarshal(cseTkgVersionsJson, &versionsMap) + if err != nil { + return result, err + } + versionMap, ok := versionsMap[parsedOvaName] + if !ok { + return result, fmt.Errorf("the Kubernetes OVA '%s' is not supported", parsedOvaName) + } + + // The map checking above guarantees that all splits and replaces will work + result.KubernetesVersion = strings.Split(parsedOvaName, "-")[0] + result.TkrVersion = strings.ReplaceAll(strings.Split(parsedOvaName, "-")[0], "+", "---") + "-" + strings.Split(parsedOvaName, "-")[1] + result.TkgVersion = versionMap.(map[string]interface{})["tkg"].(string) + result.EtcdVersion = versionMap.(map[string]interface{})["etcd"].(string) + result.CoreDnsVersion = versionMap.(map[string]interface{})["coreDns"].(string) + return result, nil +} + +// getVcdKeConfiguration gets the required information from the CSE Server configuration RDE +func getVcdKeConfiguration(vcdClient *VCDClient, cseVersion string, isNodeHealthCheckActive bool) (*vcdKeConfig, error) { + currentCseVersion := supportedCseVersions[cseVersion] + result := &vcdKeConfig{} + + rdes, err := vcdClient.GetRdesByName("vmware", "VCDKEConfig", currentCseVersion[0], "vcdKeConfig") + if err != nil { + return nil, fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", currentCseVersion[0], err) + } + if len(rdes) != 1 { + return nil, fmt.Errorf("expected exactly one VCDKEConfig RDE but got %d", len(rdes)) + } + + profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]interface{}) + if !ok { + return nil, fmt.Errorf("wrong format of VCDKEConfig, expected a 'profiles' array") + } + if len(profiles) != 1 { + return nil, fmt.Errorf("wrong format of VCDKEConfig, expected a single 'profiles' element, got %d", len(profiles)) + } + // TODO: Check airgapped environments: https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.1.1a/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.1.1/GUID-F00BE796-B5F2-48F2-A012-546E2E694400.html + result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"].(string)) + + if isNodeHealthCheckActive { + // TODO: Get the Type for this one + mhc := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"].(map[string]interface{}) + result.MaxUnhealthyNodesPercentage = mhc["maxUnhealthyNodes"].(float64) + result.NodeStartupTimeout = mhc["nodeStartupTimeout"].(string) + result.NodeNotReadyTimeout = mhc["nodeUnknownTimeout"].(string) + result.NodeUnknownTimeout = mhc["nodeNotReadyTimeout"].(string) + } + return result, nil +} + +// getRemoteFile gets a Go template file corresponding to the CSE version +func getRemoteFile(client *Client, url string) (string, error) { + resp, err := client.Http.Get(url) + if err != nil { + return "", err + } + defer func() { + if err := resp.Body.Close(); err != nil { + util.Logger.Printf("[ERROR] getRemoteFile: Could not close HTTP response body: %s", err) + } + }() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("could not get file from URL %s, got status %s", url, resp.Status) + } + + response, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return string(response), nil +} + +// getKeys retrieves all the keys from the given map and returns them as a slice +func getKeys[K comparable, V any](input map[K]V) []K { + result := make([]K, len(input)) + i := 0 + for k := range input { + result[i] = k + i++ + } + return result +} diff --git a/types/v56/cse.go b/types/v56/cse.go new file mode 100644 index 000000000..a3e49fc60 --- /dev/null +++ b/types/v56/cse.go @@ -0,0 +1,183 @@ +package types + +import "time" + +// Capvcd (Cluster API Provider for VCD), is a type that represents a Kubernetes cluster inside VCD, that is created and managed +// with the Container Service Extension (CSE) +type Capvcd struct { + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + Spec struct { + VcdKe struct { + ForceDelete bool `json:"forceDelete,omitempty"` + MarkForDelete bool `json:"markForDelete,omitempty"` + IsVCDKECluster bool `json:"isVCDKECluster,omitempty"` + AutoRepairOnErrors bool `json:"autoRepairOnErrors,omitempty"` + DefaultStorageClassOptions struct { + Filesystem string `json:"filesystem,omitempty"` + K8SStorageClassName string `json:"k8sStorageClassName,omitempty"` + VcdStorageProfileName string `json:"vcdStorageProfileName,omitempty"` + UseDeleteReclaimPolicy bool `json:"useDeleteReclaimPolicy,omitempty"` + } `json:"defaultStorageClassOptions,omitempty"` + } `json:"vcdKe,omitempty"` + CapiYaml string `json:"capiYaml,omitempty"` + } `json:"spec,omitempty"` + Status struct { + Cpi struct { + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + EventSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + AdditionalDetails struct { + DetailedEvent string `json:"Detailed Event,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"eventSet,omitempty"` + } `json:"cpi,omitempty"` + Csi struct { + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + EventSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + AdditionalDetails struct { + DetailedDescription string `json:"Detailed Description,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"eventSet,omitempty"` + } `json:"csi,omitempty"` + VcdKe struct { + State string `json:"state,omitempty"` + EventSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + AdditionalDetails struct { + DetailedEvent string `json:"Detailed Event,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"eventSet,omitempty"` + WorkerId string `json:"workerId,omitempty"` + VcdKeVersion string `json:"vcdKeVersion,omitempty"` + VcdResourceSet []struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + } `json:"vcdResourceSet,omitempty"` + HeartbeatString string `json:"heartbeatString,omitempty"` + VcdKeInstanceId string `json:"vcdKeInstanceId,omitempty"` + HeartbeatTimestamp string `json:"heartbeatTimestamp,omitempty"` + DefaultStorageClass struct { + FileSystem string `json:"fileSystem,omitempty"` + K8SStorageClassName string `json:"k8sStorageClassName,omitempty"` + VcdStorageProfileName string `json:"vcdStorageProfileName,omitempty"` + UseDeleteReclaimPolicy bool `json:"useDeleteReclaimPolicy,omitempty"` + } `json:"defaultStorageClass,omitempty"` + } `json:"vcdKe,omitempty"` + Capvcd struct { + Uid string `json:"uid,omitempty"` + Phase string `json:"phase,omitempty"` + Upgrade struct { + Ready bool `json:"ready,omitempty"` + Current struct { + TkgVersion string `json:"tkgVersion,omitempty"` + KubernetesVersion string `json:"kubernetesVersion,omitempty"` + } `json:"current,omitempty"` + } `json:"upgrade,omitempty"` + EventSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` + AdditionalDetails struct { + Event string `json:"event,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"eventSet,omitempty"` + ErrorSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` + AdditionalDetails struct { + DetailedError string `json:"Detailed Error,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"errorSet,omitempty"` + NodePool []struct { + Name string `json:"name,omitempty"` + DiskSizeMb int `json:"diskSizeMb,omitempty"` + NodeStatus map[string]interface{} `json:"nodeStatus,omitempty"` + SizingPolicy string `json:"sizingPolicy,omitempty"` + PlacementPolicy string `json:"placementPolicy,omitempty"` + NvidiaGpuEnabled bool `json:"nvidiaGpuEnabled,omitempty"` + StorageProfile string `json:"storageProfile,omitempty"` + DesiredReplicas int `json:"desiredReplicas,omitempty"` + AvailableReplicas int `json:"availableReplicas,omitempty"` + } `json:"nodePool,omitempty"` + ParentUid string `json:"parentUid,omitempty"` + K8sNetwork struct { + Pods struct { + CidrBlocks []string `json:"cidrBlocks,omitempty"` + } `json:"pods,omitempty"` + Services struct { + CidrBlocks []string `json:"cidrBlocks,omitempty"` + } `json:"services,omitempty"` + } `json:"k8sNetwork,omitempty"` + Kubernetes string `json:"kubernetes,omitempty"` + CapvcdVersion string `json:"capvcdVersion,omitempty"` + VcdProperties struct { + Site string `json:"site,omitempty"` + OrgVdcs []struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + OvdcNetworkName string `json:"ovdcNetworkName,omitempty"` + } `json:"orgVdcs,omitempty"` + Organizations []struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + } `json:"organizations,omitempty"` + } `json:"vcdProperties,omitempty"` + CapiStatusYaml string `json:"capiStatusYaml,omitempty"` + VcdResourceSet []struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + AdditionalDetails struct { + VirtualIP string `json:"virtualIP,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"vcdResourceSet,omitempty"` + ClusterApiStatus struct { + Phase string `json:"phase,omitempty"` + ApiEndpoints []struct { + Host string `json:"host,omitempty"` + Port int `json:"port,omitempty"` + } `json:"apiEndpoints,omitempty"` + } `json:"clusterApiStatus,omitempty"` + CreatedByVersion string `json:"createdByVersion,omitempty"` + ClusterResourceSetBindings []struct { + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + Applied bool `json:"applied,omitempty"` + LastAppliedTime string `json:"lastAppliedTime,omitempty"` + ClusterResourceSetName string `json:"clusterResourceSetName,omitempty"` + } `json:"clusterResourceSetBindings,omitempty"` + } `json:"capvcd,omitempty"` + Projector struct { + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + EventSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` + AdditionalDetails struct { + Event string `json:"event,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"eventSet,omitempty"` + } `json:"projector,omitempty"` + } `json:"status,omitempty"` + Metadata struct { + Name string `json:"name,omitempty"` + Site string `json:"site,omitempty"` + OrgName string `json:"orgName,omitempty"` + VirtualDataCenterName string `json:"virtualDataCenterName,omitempty"` + } `json:"metadata,omitempty"` + ApiVersion string `json:"apiVersion,omitempty"` +} From 90d2a7779aae93fb691c26aa74fa1567e7719ebd Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 26 Jan 2024 15:27:16 +0100 Subject: [PATCH 003/115] Add CSE methods Signed-off-by: abarreiro --- govcd/cse.go | 58 +++-- ...nodepool.tmpl => capiyaml_workerpool.tmpl} | 0 govcd/cse_template.go | 206 ++++++++++++++++++ govcd/cse_util.go | 94 ++++++-- 4 files changed, 314 insertions(+), 44 deletions(-) rename govcd/cse/4.2/{capiyaml_nodepool.tmpl => capiyaml_workerpool.tmpl} (100%) create mode 100644 govcd/cse_template.go diff --git a/govcd/cse.go b/govcd/cse.go index c8b0a9ba2..499ceb857 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -46,6 +46,11 @@ type CseClusterCreationInput struct { Owner string // Optional, if not set will pick the current user present in the VCDClient ApiToken string NodeHealthCheck bool + PodCidr string + ServiceCidr string + SshPublicKey string + VirtualIpSubnet string + AutoRepairOnErrors bool } // ControlPlaneInput defines the required elements that the consumer of these Container Service Extension (CSE) methods @@ -88,26 +93,34 @@ var cseTkgVersionsJson []byte // it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster in the returned CseClusterApiProviderCluster. // If the cluster is created correctly, returns all the data in CseClusterApiProviderCluster. func (vcdClient *VCDClient) CseCreateKubernetesCluster(clusterData CseClusterCreationInput, timeoutMinutes time.Duration) (*CseClusterApiProviderCluster, error) { - _, err := getRemoteFile(&vcdClient.Client, fmt.Sprintf("%s/capiyaml_cluster.tmpl", clusterData.CseVersion)) + goTemplateContents, err := clusterData.toCseClusterCreationGoTemplateContents(vcdClient) if err != nil { return nil, err } - _, err = clusterData.toCseClusterCreationPayload(vcdClient) + rdeContents, err := getCseKubernetesClusterCreationPayload(vcdClient, goTemplateContents) if err != nil { return nil, err } - err = waitUntilClusterIsProvisioned(vcdClient, "", timeoutMinutes) + rde, err := vcdClient.CreateRde("vmware", "capvcdCluster", supportedCseVersions[clusterData.CseVersion][1], types.DefinedEntity{ + EntityType: goTemplateContents.RdeType.ID, + Name: goTemplateContents.Name, + Entity: rdeContents, + }, &TenantContext{ + OrgId: goTemplateContents.OrganizationId, + OrgName: goTemplateContents.OrganizationName, + }) if err != nil { return nil, err } - result := &CseClusterApiProviderCluster{ - client: &vcdClient.Client, - ID: "", + cluster, err := waitUntilClusterIsProvisioned(vcdClient, rde.DefinedEntity.ID, timeoutMinutes) + if err != nil { + return nil, err } - return result, nil + + return cluster, nil } // CseConvertToCapvcdCluster takes the receiver, which is a generic RDE that must represent an existing CSE Kubernetes cluster, @@ -128,6 +141,7 @@ func (rde *DefinedEntity) CseConvertToCapvcdCluster() (*CseClusterApiProviderClu result := &CseClusterApiProviderCluster{ Capvcd: &types.Capvcd{}, + ID: rde.DefinedEntity.ID, Etag: rde.Etag, client: rde.client, } @@ -139,10 +153,13 @@ func (rde *DefinedEntity) CseConvertToCapvcdCluster() (*CseClusterApiProviderClu return result, nil } -// waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if "operations_timeout_minutes=0") -// or until this timeout is reached. If one of the states is "error", this function also checks whether "auto_repair_on_errors=true" to keep -// waiting. -func waitUntilClusterIsProvisioned(vcdClient *VCDClient, clusterId string, timeoutMinutes time.Duration) error { +// waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if timeoutMinutes = 0) +// or until this timeout is reached. If the cluster is in "provisioned" state before the given timeout, it returns a CseClusterApiProviderCluster object +// representing the Kubernetes cluster with all the latest information. +// If one of the states of the cluster at a given point is "error", this function also checks whether the cluster has the "Auto Repair on Errors" flag enabled, +// so it keeps waiting if it's true. +// If timeout is reached before the cluster, it returns an error. +func waitUntilClusterIsProvisioned(vcdClient *VCDClient, clusterId string, timeoutMinutes time.Duration) (*CseClusterApiProviderCluster, error) { var elapsed time.Duration logHttpResponse := util.LogHttpResponse sleepTime := 30 @@ -154,35 +171,34 @@ func waitUntilClusterIsProvisioned(vcdClient *VCDClient, clusterId string, timeo }() start := time.Now() - latestState := "" + var capvcdCluster *CseClusterApiProviderCluster for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies operations_timeout_minutes=0, we wait forever util.LogHttpResponse = false rde, err := vcdClient.GetRdeById(clusterId) util.LogHttpResponse = logHttpResponse if err != nil { - return err + return nil, err } - capvcdCluster, err := rde.CseConvertToCapvcdCluster() + capvcdCluster, err = rde.CseConvertToCapvcdCluster() if err != nil { - return err + return nil, err } - latestState = capvcdCluster.Capvcd.Status.VcdKe.State - switch latestState { + switch capvcdCluster.Capvcd.Status.VcdKe.State { case "provisioned": - return nil + return capvcdCluster, nil case "error": // We just finish if auto-recovery is disabled, otherwise we just let CSE fixing things in background if !capvcdCluster.Capvcd.Spec.VcdKe.AutoRepairOnErrors { // Try to give feedback about what went wrong, which is located in a set of events in the RDE payload - return fmt.Errorf("got an error and 'auto_repair_on_errors=false', aborting. Errors: %s", capvcdCluster.Capvcd.Status.Capvcd.ErrorSet[len(capvcdCluster.Capvcd.Status.Capvcd.ErrorSet)-1].AdditionalDetails.DetailedError) + return capvcdCluster, fmt.Errorf("got an error and 'auto repair on errors' is disabled, aborting. Errors: %s", capvcdCluster.Capvcd.Status.Capvcd.ErrorSet[len(capvcdCluster.Capvcd.Status.Capvcd.ErrorSet)-1].AdditionalDetails.DetailedError) } } - util.Logger.Printf("[DEBUG] Cluster '%s' is in '%s' state, will check again in %d seconds", capvcdCluster.ID, latestState, sleepTime) + util.Logger.Printf("[DEBUG] Cluster '%s' is in '%s' state, will check again in %d seconds", capvcdCluster.ID, capvcdCluster.Capvcd.Status.VcdKe.State, sleepTime) elapsed = time.Since(start) time.Sleep(time.Duration(sleepTime) * time.Second) } - return fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, latestState) + return capvcdCluster, fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, capvcdCluster.Capvcd.Status.VcdKe.State) } diff --git a/govcd/cse/4.2/capiyaml_nodepool.tmpl b/govcd/cse/4.2/capiyaml_workerpool.tmpl similarity index 100% rename from govcd/cse/4.2/capiyaml_nodepool.tmpl rename to govcd/cse/4.2/capiyaml_workerpool.tmpl diff --git a/govcd/cse_template.go b/govcd/cse_template.go new file mode 100644 index 000000000..7937add87 --- /dev/null +++ b/govcd/cse_template.go @@ -0,0 +1,206 @@ +package govcd + +import ( + "bytes" + _ "embed" + "encoding/base64" + "encoding/json" + "fmt" + "strconv" + "strings" + "text/template" +) + +// getCseKubernetesClusterCreationPayload gets the payload for the RDE that will trigger a Kubernetes cluster creation. +// It generates a valid YAML that is embedded inside the RDE JSON, then it is returned as an unmarshaled +// generic map, that allows to be sent to VCD as it is. +func getCseKubernetesClusterCreationPayload(vcdClient *VCDClient, goTemplateContents *cseClusterCreationGoTemplateArguments) (map[string]interface{}, error) { + capiYaml, err := generateCapiYaml(vcdClient, goTemplateContents) + if err != nil { + return nil, err + } + + args := map[string]string{ + "Name": goTemplateContents.Name, + "Org": goTemplateContents.OrganizationName, + "VcdUrl": goTemplateContents.VcdUrl, + "Vdc": goTemplateContents.VdcName, + "Delete": "false", + "ForceDelete": "false", + "AutoRepairOnErrors": strconv.FormatBool(goTemplateContents.AutoRepairOnErrors), + "ApiToken": goTemplateContents.ApiToken, + "CapiYaml": capiYaml, + } + + if goTemplateContents.DefaultStorageClass.StorageProfileName != "" { + args["DefaultStorageClassStorageProfile"] = goTemplateContents.DefaultStorageClass.StorageProfileName + args["DefaultStorageClassName"] = goTemplateContents.DefaultStorageClass.Name + args["DefaultStorageClassUseDeleteReclaimPolicy"] = strconv.FormatBool(goTemplateContents.DefaultStorageClass.UseDeleteReclaimPolicy) + args["DefaultStorageClassFileSystem"] = goTemplateContents.DefaultStorageClass.Filesystem + } + + rdeTmpl, err := getRemoteFile(&vcdClient.Client, "rde") + if err != nil { + return nil, err + } + + capvcdEmpty := template.Must(template.New(goTemplateContents.Name).Parse(rdeTmpl)) + buf := &bytes.Buffer{} + if err := capvcdEmpty.Execute(buf, args); err != nil { + return nil, fmt.Errorf("could not render the Go template with the CAPVCD JSON: %s", err) + } + + var result interface{} + err = json.Unmarshal(buf.Bytes(), &result) + if err != nil { + return nil, fmt.Errorf("could not generate a correct CAPVCD JSON: %s", err) + } + + return result.(map[string]interface{}), nil +} + +// generateNodePoolYaml generates YAML blocks corresponding to the Kubernetes node pools. +func generateNodePoolYaml(vcdClient *VCDClient, clusterDetails *cseClusterCreationGoTemplateArguments) (string, error) { + workerPoolTmpl, err := getRemoteFile(&vcdClient.Client, "capiyaml_workerpool") + if err != nil { + return "", err + } + + nodePoolEmptyTmpl := template.Must(template.New(clusterDetails.Name + "-worker-pool").Parse(workerPoolTmpl)) + resultYaml := "" + buf := &bytes.Buffer{} + + // We can have many worker pools, we build a YAML object for each one of them. + for _, workerPool := range clusterDetails.WorkerPools { + + // Check the correctness of the compute policies in the node pool block + if workerPool.PlacementPolicyName != "" && workerPool.VGpuPolicyName != "" { + return "", fmt.Errorf("the worker pool '%s' should have either a Placement Policy or a vGPU Policy, not both", workerPool.Name) + } + placementPolicy := workerPool.PlacementPolicyName + if workerPool.VGpuPolicyName != "" { + placementPolicy = workerPool.VGpuPolicyName // For convenience, we just use one of the variables as both cannot be set at same time + } + + if err := nodePoolEmptyTmpl.Execute(buf, map[string]string{ + "ClusterName": clusterDetails.Name, + "NodePoolName": workerPool.Name, + "TargetNamespace": clusterDetails.Name + "-ns", + "Catalog": clusterDetails.CatalogName, + "VAppTemplate": clusterDetails.KubernetesTemplateOvaName, + "NodePoolSizingPolicy": workerPool.SizingPolicyName, + "NodePoolPlacementPolicy": placementPolicy, // Can be either Placement or vGPU policy + "NodePoolStorageProfile": workerPool.StorageProfileName, + "NodePoolDiskSize": fmt.Sprintf("%dGi", workerPool.DiskSizeGi), + "NodePoolEnableGpu": strconv.FormatBool(workerPool.VGpuPolicyName != ""), + "NodePoolMachineCount": strconv.Itoa(workerPool.MachineCount), + "KubernetesVersion": clusterDetails.TkgVersionBundle.KubernetesVersion, + }); err != nil { + return "", fmt.Errorf("could not generate a correct Node Pool YAML: %s", err) + } + resultYaml += fmt.Sprintf("%s\n---\n", buf.String()) + buf.Reset() + } + return resultYaml, nil +} + +// generateMemoryHealthCheckYaml generates a YAML block corresponding to the Kubernetes memory health check. +func generateMemoryHealthCheckYaml(vcdClient *VCDClient, clusterDetails *cseClusterCreationGoTemplateArguments, clusterName string) (string, error) { + if clusterDetails.MachineHealthCheck == nil { + return "", nil + } + + mhcTmpl, err := getRemoteFile(&vcdClient.Client, "capiyaml_mhc") + if err != nil { + return "", err + } + + mhcEmptyTmpl := template.Must(template.New(clusterName + "-mhc").Parse(mhcTmpl)) + buf := &bytes.Buffer{} + + if err := mhcEmptyTmpl.Execute(buf, map[string]string{ + "ClusterName": clusterName, + "TargetNamespace": clusterName + "-ns", + "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", clusterDetails.MachineHealthCheck.MaxUnhealthyNodesPercentage), // With the 'percentage' suffix + "NodeStartupTimeout": fmt.Sprintf("%ss", clusterDetails.MachineHealthCheck.NodeStartupTimeout), // With the 'second' suffix + "NodeUnknownTimeout": fmt.Sprintf("%ss", clusterDetails.MachineHealthCheck.NodeUnknownTimeout), // With the 'second' suffix + "NodeNotReadyTimeout": fmt.Sprintf("%ss", clusterDetails.MachineHealthCheck.NodeNotReadyTimeout), // With the 'second' suffix + }); err != nil { + return "", fmt.Errorf("could not generate a correct Memory Health Check YAML: %s", err) + } + return fmt.Sprintf("%s\n---\n", buf.String()), nil + +} + +// generateCapiYaml generates the YAML string that is required during Kubernetes cluster creation, to be embedded +// in the CAPVCD cluster JSON payload. This function picks data from the Terraform schema and the createClusterDto to +// populate several Go templates and build a final YAML. +func generateCapiYaml(vcdClient *VCDClient, clusterDetails *cseClusterCreationGoTemplateArguments) (string, error) { + clusterTmpl, err := getRemoteFile(&vcdClient.Client, "capiyaml_cluster") + if err != nil { + return "", err + } + + // This YAML snippet contains special strings, such as "%,", that render wrong using the Go template engine + sanitizedTemplate := strings.NewReplacer("%", "%%").Replace(clusterTmpl) + capiYamlEmpty := template.Must(template.New(clusterDetails.Name + "-cluster").Parse(sanitizedTemplate)) + + nodePoolYaml, err := generateNodePoolYaml(vcdClient, clusterDetails) + if err != nil { + return "", err + } + + memoryHealthCheckYaml, err := generateMemoryHealthCheckYaml(vcdClient, clusterDetails, clusterDetails.Name) + if err != nil { + return "", err + } + + args := map[string]string{ + "ClusterName": clusterDetails.Name, + "TargetNamespace": clusterDetails.Name + "-ns", + "TkrVersion": clusterDetails.TkgVersionBundle.TkrVersion, + "TkgVersion": clusterDetails.TkgVersionBundle.TkgVersion, + "UsernameB64": base64.StdEncoding.EncodeToString([]byte(clusterDetails.Owner)), + "ApiTokenB64": base64.StdEncoding.EncodeToString([]byte(clusterDetails.ApiToken)), + "PodCidr": clusterDetails.PodCidr, + "ServiceCidr": clusterDetails.ServiceCidr, + "VcdSite": clusterDetails.VcdUrl, + "Org": clusterDetails.OrganizationName, + "OrgVdc": clusterDetails.VdcName, + "OrgVdcNetwork": clusterDetails.NetworkName, + "Catalog": clusterDetails.CatalogName, + "VAppTemplate": clusterDetails.KubernetesTemplateOvaName, + "ControlPlaneSizingPolicy": clusterDetails.ControlPlane.SizingPolicyName, + "ControlPlanePlacementPolicy": clusterDetails.ControlPlane.PlacementPolicyName, + "ControlPlaneStorageProfile": clusterDetails.ControlPlane.StorageProfileName, + "ControlPlaneDiskSize": fmt.Sprintf("%dGi", clusterDetails.ControlPlane.DiskSizeGi), + "ControlPlaneMachineCount": strconv.Itoa(clusterDetails.ControlPlane.MachineCount), + "ControlPlaneEndpoint": clusterDetails.ControlPlane.Ip, + "DnsVersion": clusterDetails.TkgVersionBundle.CoreDnsVersion, + "EtcdVersion": clusterDetails.TkgVersionBundle.EtcdVersion, + "ContainerRegistryUrl": clusterDetails.ContainerRegistryUrl, + "KubernetesVersion": clusterDetails.TkgVersionBundle.KubernetesVersion, + "SshPublicKey": clusterDetails.SshPublicKey, + "VirtualIpSubnet": clusterDetails.VirtualIpSubnet, + } + + buf := &bytes.Buffer{} + if err := capiYamlEmpty.Execute(buf, args); err != nil { + return "", fmt.Errorf("could not generate a correct CAPI YAML: %s", err) + } + // The final "pretty" YAML. To embed it in the final payload it must be marshaled into a one-line JSON string + prettyYaml := fmt.Sprintf("%s\n%s\n%s", memoryHealthCheckYaml, nodePoolYaml, buf.String()) + + // We don't use a standard json.Marshal() as the YAML contains special + // characters that are not encoded properly, such as '<'. + buf.Reset() + enc := json.NewEncoder(buf) + enc.SetEscapeHTML(false) + err = enc.Encode(prettyYaml) + if err != nil { + return "", fmt.Errorf("could not encode the CAPI YAML into JSON: %s", err) + } + + // Removes trailing quotes from the final JSON string + return strings.Trim(strings.TrimSpace(buf.String()), "\""), nil +} diff --git a/govcd/cse_util.go b/govcd/cse_util.go index a63667ec5..bb5c8261e 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -14,10 +14,11 @@ import ( // cseClusterCreationGoTemplateArguments defines the required arguments that are required by the Go templates used internally to specify // a Kubernetes cluster. These are not set by the user, but instead they are computed from a valid -// CseClusterCreationInput object in the CseClusterCreationInput.toCseClusterCreationPayload method. These fields are then +// CseClusterCreationInput object in the CseClusterCreationInput.toCseClusterCreationGoTemplateContents method. These fields are then // inserted in Go templates to render a final JSON that is valid to be used as the cluster Runtime Defined Entity (RDE) payload. type cseClusterCreationGoTemplateArguments struct { Name string + OrganizationId string OrganizationName string VdcName string NetworkName string @@ -28,10 +29,16 @@ type cseClusterCreationGoTemplateArguments struct { ControlPlane controlPlane WorkerPools []workerPool DefaultStorageClass defaultStorageClass - VcdKeConfig *vcdKeConfig + MachineHealthCheck *machineHealthCheck Owner string ApiToken string VcdUrl string + ContainerRegistryUrl string + VirtualIpSubnet string + SshPublicKey string + PodCidr string + ServiceCidr string + AutoRepairOnErrors bool } // controlPlane defines the Control Plane inside cseClusterCreationGoTemplateArguments @@ -57,20 +64,19 @@ type workerPool struct { // defaultStorageClass defines a Default Storage Class inside cseClusterCreationGoTemplateArguments type defaultStorageClass struct { - StorageProfileName string - Name string - ReclaimPolicy string - Filesystem string + StorageProfileName string + Name string + UseDeleteReclaimPolicy bool + Filesystem string } -// vcdKeConfig is a type that contains only the required and relevant fields from the Container Service Extension (CSE) installation configuration, +// machineHealthCheck is a type that contains only the required and relevant fields from the Container Service Extension (CSE) installation configuration, // such as the Machine Health Check settings or the container registry URL. -type vcdKeConfig struct { +type machineHealthCheck struct { MaxUnhealthyNodesPercentage float64 NodeStartupTimeout string NodeNotReadyTimeout string NodeUnknownTimeout string - ContainerRegistryUrl string } // validate validates the CSE Kubernetes cluster creation input data. Returns an error if some of the fields is wrong. @@ -140,9 +146,9 @@ func (ccd *CseClusterCreationInput) validate() error { return nil } -// toCseClusterCreationPayload transforms user input data (receiver CseClusterCreationInput) into the final payload that +// toCseClusterCreationGoTemplateContents transforms user input data (receiver CseClusterCreationInput) into the final payload that // will be used to render the Go templates that define a Kubernetes cluster creation payload (cseClusterCreationGoTemplateArguments). -func (input *CseClusterCreationInput) toCseClusterCreationPayload(vcdClient *VCDClient) (*cseClusterCreationGoTemplateArguments, error) { +func (input *CseClusterCreationInput) toCseClusterCreationGoTemplateContents(vcdClient *VCDClient) (*cseClusterCreationGoTemplateArguments, error) { output := &cseClusterCreationGoTemplateArguments{} err := input.validate() if err != nil { @@ -152,6 +158,7 @@ func (input *CseClusterCreationInput) toCseClusterCreationPayload(vcdClient *VCD if err != nil { return nil, fmt.Errorf("could not retrieve the Organization with ID '%s': %s", input.VdcId, err) } + output.OrganizationId = org.Org.ID output.OrganizationName = org.Org.Name vdc, err := org.GetVDCById(input.VdcId, true) @@ -243,22 +250,35 @@ func (input *CseClusterCreationInput) toCseClusterCreationPayload(vcdClient *VCD SizingPolicyName: idToNameCache[input.ControlPlane.SizingPolicyId], PlacementPolicyName: idToNameCache[input.ControlPlane.PlacementPolicyId], StorageProfileName: idToNameCache[input.ControlPlane.StorageProfileId], + Ip: input.ControlPlane.Ip, } if input.DefaultStorageClass != nil { output.DefaultStorageClass = defaultStorageClass{ StorageProfileName: idToNameCache[input.DefaultStorageClass.StorageProfileId], Name: input.DefaultStorageClass.Name, - ReclaimPolicy: input.DefaultStorageClass.ReclaimPolicy, Filesystem: input.DefaultStorageClass.Filesystem, } + output.DefaultStorageClass.UseDeleteReclaimPolicy = false + if input.DefaultStorageClass.ReclaimPolicy == "delete" { + output.DefaultStorageClass.UseDeleteReclaimPolicy = true + } } - vcdKeConfig, err := getVcdKeConfiguration(vcdClient, input.CseVersion, input.NodeHealthCheck) + mhc, err := getMachineHealthCheck(vcdClient, input.CseVersion, input.NodeHealthCheck) if err != nil { return nil, err } - output.VcdKeConfig = vcdKeConfig + if mhc != nil { + output.MachineHealthCheck = mhc + } + + containerRegistryUrl, err := getContainerRegistryUrl(vcdClient, input.CseVersion) + if err != nil { + return nil, err + } + + output.ContainerRegistryUrl = containerRegistryUrl output.Owner = input.Owner if input.Owner == "" { @@ -270,6 +290,13 @@ func (input *CseClusterCreationInput) toCseClusterCreationPayload(vcdClient *VCD } output.VcdUrl = strings.Replace(vcdClient.Client.VCDHREF.String(), "/api", "", 1) + // These don't change, don't need mapping + output.PodCidr = input.PodCidr + output.ServiceCidr = input.ServiceCidr + output.SshPublicKey = input.SshPublicKey + output.VirtualIpSubnet = input.VirtualIpSubnet + output.AutoRepairOnErrors = input.AutoRepairOnErrors + return output, nil } @@ -320,19 +347,19 @@ func getTkgVersionBundleFromVAppTemplateName(ovaName string) (tkgVersionBundle, return result, nil } -// getVcdKeConfiguration gets the required information from the CSE Server configuration RDE -func getVcdKeConfiguration(vcdClient *VCDClient, cseVersion string, isNodeHealthCheckActive bool) (*vcdKeConfig, error) { +// getMachineHealthCheck gets the required information from the CSE Server configuration RDE +func getMachineHealthCheck(vcdClient *VCDClient, cseVersion string, isNodeHealthCheckActive bool) (*machineHealthCheck, error) { currentCseVersion := supportedCseVersions[cseVersion] - result := &vcdKeConfig{} + result := machineHealthCheck{} - rdes, err := vcdClient.GetRdesByName("vmware", "VCDKEConfig", currentCseVersion[0], "vcdKeConfig") + rdes, err := vcdClient.GetRdesByName("vmware", "VCDKEConfig", currentCseVersion[0], "machineHealthCheck") if err != nil { return nil, fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", currentCseVersion[0], err) } if len(rdes) != 1 { return nil, fmt.Errorf("expected exactly one VCDKEConfig RDE but got %d", len(rdes)) } - + // TODO: Get the struct Type for this one profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]interface{}) if !ok { return nil, fmt.Errorf("wrong format of VCDKEConfig, expected a 'profiles' array") @@ -340,18 +367,39 @@ func getVcdKeConfiguration(vcdClient *VCDClient, cseVersion string, isNodeHealth if len(profiles) != 1 { return nil, fmt.Errorf("wrong format of VCDKEConfig, expected a single 'profiles' element, got %d", len(profiles)) } - // TODO: Check airgapped environments: https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.1.1a/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.1.1/GUID-F00BE796-B5F2-48F2-A012-546E2E694400.html - result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"].(string)) if isNodeHealthCheckActive { - // TODO: Get the Type for this one + // TODO: Get the struct Type for this one mhc := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"].(map[string]interface{}) result.MaxUnhealthyNodesPercentage = mhc["maxUnhealthyNodes"].(float64) result.NodeStartupTimeout = mhc["nodeStartupTimeout"].(string) result.NodeNotReadyTimeout = mhc["nodeUnknownTimeout"].(string) result.NodeUnknownTimeout = mhc["nodeNotReadyTimeout"].(string) } - return result, nil + return nil, nil +} + +// getContainerRegistryUrl gets the required information from the CSE Server configuration RDE +func getContainerRegistryUrl(vcdClient *VCDClient, cseVersion string) (string, error) { + currentCseVersion := supportedCseVersions[cseVersion] + + rdes, err := vcdClient.GetRdesByName("vmware", "VCDKEConfig", currentCseVersion[0], "machineHealthCheck") + if err != nil { + return "", fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", currentCseVersion[0], err) + } + if len(rdes) != 1 { + return "", fmt.Errorf("expected exactly one VCDKEConfig RDE but got %d", len(rdes)) + } + // TODO: Get the struct Type for this one + profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]interface{}) + if !ok { + return "", fmt.Errorf("wrong format of VCDKEConfig, expected a 'profiles' array") + } + if len(profiles) != 1 { + return "", fmt.Errorf("wrong format of VCDKEConfig, expected a single 'profiles' element, got %d", len(profiles)) + } + // TODO: Check airgapped environments: https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.1.1a/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.1.1/GUID-F00BE796-B5F2-48F2-A012-546E2E694400.html + return fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"].(string)), nil } // getRemoteFile gets a Go template file corresponding to the CSE version From 259cde486145ecb7893bb73645b856c35a414cc1 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 26 Jan 2024 16:13:42 +0100 Subject: [PATCH 004/115] Start testing create Signed-off-by: abarreiro --- govcd/api_vcd_test.go | 11 +++- govcd/common_test.go | 2 +- govcd/cse_template.go | 8 +-- govcd/cse_test.go | 90 +++++++++++++++++++++++++++++ govcd/cse_util.go | 17 +++++- govcd/sample_govcd_test_config.yaml | 15 +++++ 6 files changed, 135 insertions(+), 8 deletions(-) diff --git a/govcd/api_vcd_test.go b/govcd/api_vcd_test.go index 73ce87cb8..7d3220636 100644 --- a/govcd/api_vcd_test.go +++ b/govcd/api_vcd_test.go @@ -1,4 +1,4 @@ -//go:build api || openapi || functional || catalog || vapp || gateway || network || org || query || extnetwork || task || vm || vdc || system || disk || lb || lbAppRule || lbAppProfile || lbServerPool || lbServiceMonitor || lbVirtualServer || user || search || nsxv || nsxt || auth || affinity || role || alb || certificate || vdcGroup || metadata || providervdc || rde || vsphere || uiPlugin || ALL +//go:build api || openapi || functional || catalog || vapp || gateway || network || org || query || extnetwork || task || vm || vdc || system || disk || lb || lbAppRule || lbAppProfile || lbServerPool || lbServiceMonitor || lbVirtualServer || user || search || nsxv || nsxt || auth || affinity || role || alb || certificate || vdcGroup || metadata || providervdc || rde || vsphere || uiPlugin || cse || ALL /* * Copyright 2022 VMware, Inc. All rights reserved. Licensed under the Apache v2 License. @@ -239,6 +239,15 @@ type TestConfig struct { MediaUdfTypePath string `yaml:"mediaUdfTypePath,omitempty"` UiPluginPath string `yaml:"uiPluginPath,omitempty"` } `yaml:"media"` + Cse struct { + SolutionsOrg string `yaml:"solutionsOrg,omitempty"` + TenantOrg string `yaml:"tenantOrg,omitempty"` + TenantVdc string `yaml:"tenantVdc,omitempty"` + RoutedNetwork string `yaml:"routedNetwork,omitempty"` + EdgeGateway string `yaml:"edgeGateway,omitempty"` + OvaCatalog string `yaml:"ovaCatalog,omitempty"` + OvaName string `yaml:"ovaName,omitempty"` + } `yaml:"cse"` } // Test struct for vcloud-director. diff --git a/govcd/common_test.go b/govcd/common_test.go index 20e8c613e..eba4a5753 100644 --- a/govcd/common_test.go +++ b/govcd/common_test.go @@ -1,4 +1,4 @@ -//go:build api || auth || functional || catalog || vapp || gateway || network || org || query || extnetwork || task || vm || vdc || system || disk || lb || lbAppRule || lbAppProfile || lbServerPool || lbServiceMonitor || lbVirtualServer || user || role || nsxv || nsxt || openapi || affinity || search || alb || certificate || vdcGroup || metadata || providervdc || rde || uiPlugin || vsphere || ALL +//go:build api || auth || functional || catalog || vapp || gateway || network || org || query || extnetwork || task || vm || vdc || system || disk || lb || lbAppRule || lbAppProfile || lbServerPool || lbServiceMonitor || lbVirtualServer || user || role || nsxv || nsxt || openapi || affinity || search || alb || certificate || vdcGroup || metadata || providervdc || rde || uiPlugin || vsphere || cse || ALL /* * Copyright 2021 VMware, Inc. All rights reserved. Licensed under the Apache v2 License. diff --git a/govcd/cse_template.go b/govcd/cse_template.go index 7937add87..a0342cde6 100644 --- a/govcd/cse_template.go +++ b/govcd/cse_template.go @@ -39,7 +39,7 @@ func getCseKubernetesClusterCreationPayload(vcdClient *VCDClient, goTemplateCont args["DefaultStorageClassFileSystem"] = goTemplateContents.DefaultStorageClass.Filesystem } - rdeTmpl, err := getRemoteFile(&vcdClient.Client, "rde") + rdeTmpl, err := getCseTemplate(&vcdClient.Client, goTemplateContents.CseVersion, "rde") if err != nil { return nil, err } @@ -61,7 +61,7 @@ func getCseKubernetesClusterCreationPayload(vcdClient *VCDClient, goTemplateCont // generateNodePoolYaml generates YAML blocks corresponding to the Kubernetes node pools. func generateNodePoolYaml(vcdClient *VCDClient, clusterDetails *cseClusterCreationGoTemplateArguments) (string, error) { - workerPoolTmpl, err := getRemoteFile(&vcdClient.Client, "capiyaml_workerpool") + workerPoolTmpl, err := getCseTemplate(&vcdClient.Client, clusterDetails.CseVersion, "capiyaml_workerpool") if err != nil { return "", err } @@ -110,7 +110,7 @@ func generateMemoryHealthCheckYaml(vcdClient *VCDClient, clusterDetails *cseClus return "", nil } - mhcTmpl, err := getRemoteFile(&vcdClient.Client, "capiyaml_mhc") + mhcTmpl, err := getCseTemplate(&vcdClient.Client, clusterDetails.CseVersion, "capiyaml_mhc") if err != nil { return "", err } @@ -136,7 +136,7 @@ func generateMemoryHealthCheckYaml(vcdClient *VCDClient, clusterDetails *cseClus // in the CAPVCD cluster JSON payload. This function picks data from the Terraform schema and the createClusterDto to // populate several Go templates and build a final YAML. func generateCapiYaml(vcdClient *VCDClient, clusterDetails *cseClusterCreationGoTemplateArguments) (string, error) { - clusterTmpl, err := getRemoteFile(&vcdClient.Client, "capiyaml_cluster") + clusterTmpl, err := getCseTemplate(&vcdClient.Client, clusterDetails.CseVersion, "capiyaml_cluster") if err != nil { return "", err } diff --git a/govcd/cse_test.go b/govcd/cse_test.go index a7a126376..268a07c6f 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -8,13 +8,103 @@ package govcd import ( "fmt" + "github.com/vmware/go-vcloud-director/v2/types/v56" . "gopkg.in/check.v1" + "net/url" + "os" ) +const ( + TestRequiresCseConfiguration = "Test %s requires CSE configuration details" +) + +func skipCseTests(testConfig TestConfig) bool { + if cse := os.Getenv("TEST_VCD_CSE"); cse == "" { + return true + } + return testConfig.Cse.SolutionsOrg == "" || testConfig.Cse.TenantOrg == "" || testConfig.Cse.OvaName == "" || + testConfig.Cse.RoutedNetwork == "" || testConfig.Cse.EdgeGateway == "" || testConfig.Cse.OvaCatalog == "" || testConfig.Cse.TenantVdc == "" || + testConfig.VCD.StorageProfile.SP1 == "" +} + // Test_Cse func (vcd *TestVCD) Test_Cse(check *C) { if vcd.skipAdminTests { check.Skip(fmt.Sprintf(TestRequiresSysAdminPrivileges, check.TestName())) } + if skipCseTests(vcd.config) { + check.Skip(fmt.Sprintf(TestRequiresCseConfiguration, check.TestName())) + } + + org, err := vcd.client.GetOrgByName(vcd.config.Cse.TenantOrg) + check.Assert(err, IsNil) + + vdc, err := org.GetVDCByName(vcd.config.Cse.TenantOrg, false) + check.Assert(err, IsNil) + + net, err := vdc.GetOrgVdcNetworkByName(vcd.config.Cse.RoutedNetwork, false) + check.Assert(err, IsNil) + + ova, err := vdc.GetVAppTemplateByName(vcd.config.Cse.OvaName) + check.Assert(err, IsNil) + + sp, err := vdc.FindStorageProfileReference(vcd.config.VCD.StorageProfile.SP1) + check.Assert(err, IsNil) + + policies, err := vcd.client.GetAllVdcComputePoliciesV2(url.Values{ + "filter": []string{"name==*TKG%20small*"}, + }) + check.Assert(err, IsNil) + check.Assert(len(policies), Equals, 1) + + token, err := vcd.client.CreateToken(vcd.config.Provider.SysOrg, check.TestName()) + check.Assert(err, IsNil) + AddToCleanupListOpenApi(token.Token.Name, check.TestName(), types.OpenApiPathVersion1_0_0+types.OpenApiEndpointTokens+token.Token.ID) + + apiToken, err := token.GetInitialApiToken() + check.Assert(err, IsNil) + + cluster, err := vcd.client.CseCreateKubernetesCluster(CseClusterCreationInput{ + Name: "test-cse", + OrganizationId: org.Org.ID, + VdcId: vdc.Vdc.ID, + NetworkId: net.OrgVDCNetwork.ID, + KubernetesTemplateOvaId: ova.VAppTemplate.ID, + CseVersion: "4.2", + ControlPlane: ControlPlaneInput{ + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyId: policies[0].VdcComputePolicyV2.ID, + StorageProfileId: sp.ID, + Ip: "", + }, + WorkerPools: []WorkerPoolInput{{ + Name: "worker-pool-1", + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyId: policies[0].VdcComputePolicyV2.ID, + StorageProfileId: sp.ID, + }}, + DefaultStorageClass: &DefaultStorageClassInput{ + StorageProfileId: sp.ID, + Name: "storage-class-1", + ReclaimPolicy: "delete", + Filesystem: "ext4", + }, + Owner: vcd.config.Provider.User, + ApiToken: apiToken.RefreshToken, + NodeHealthCheck: true, + PodCidr: "100.96.0.0/11", + ServiceCidr: "100.64.0.0/13", + AutoRepairOnErrors: true, + }, 0) + check.Assert(err, IsNil) + check.Assert(cluster.ID, Not(Equals), "") + check.Assert(cluster.Etag, Not(Equals), "") + check.Assert(cluster.Capvcd.Status.VcdKe.State, Equals, "provisioned") + + err = token.Delete() + check.Assert(err, IsNil) + } diff --git a/govcd/cse_util.go b/govcd/cse_util.go index bb5c8261e..8644c0f6f 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -17,6 +17,7 @@ import ( // CseClusterCreationInput object in the CseClusterCreationInput.toCseClusterCreationGoTemplateContents method. These fields are then // inserted in Go templates to render a final JSON that is valid to be used as the cluster Runtime Defined Entity (RDE) payload. type cseClusterCreationGoTemplateArguments struct { + CseVersion string Name string OrganizationId string OrganizationName string @@ -142,6 +143,12 @@ func (ccd *CseClusterCreationInput) validate() error { if ccd.ApiToken == "" { return fmt.Errorf("the API token is required") } + if ccd.PodCidr == "" { + return fmt.Errorf("the Pod CIDR is required") + } + if ccd.ServiceCidr == "" { + return fmt.Errorf("the Service CIDR is required") + } return nil } @@ -149,11 +156,12 @@ func (ccd *CseClusterCreationInput) validate() error { // toCseClusterCreationGoTemplateContents transforms user input data (receiver CseClusterCreationInput) into the final payload that // will be used to render the Go templates that define a Kubernetes cluster creation payload (cseClusterCreationGoTemplateArguments). func (input *CseClusterCreationInput) toCseClusterCreationGoTemplateContents(vcdClient *VCDClient) (*cseClusterCreationGoTemplateArguments, error) { - output := &cseClusterCreationGoTemplateArguments{} err := input.validate() if err != nil { return nil, err } + + output := &cseClusterCreationGoTemplateArguments{} org, err := vcdClient.GetOrgById(input.OrganizationId) if err != nil { return nil, fmt.Errorf("could not retrieve the Organization with ID '%s': %s", input.VdcId, err) @@ -296,7 +304,7 @@ func (input *CseClusterCreationInput) toCseClusterCreationGoTemplateContents(vcd output.SshPublicKey = input.SshPublicKey output.VirtualIpSubnet = input.VirtualIpSubnet output.AutoRepairOnErrors = input.AutoRepairOnErrors - + output.CseVersion = input.CseVersion return output, nil } @@ -402,8 +410,13 @@ func getContainerRegistryUrl(vcdClient *VCDClient, cseVersion string) (string, e return fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"].(string)), nil } +func getCseTemplate(client *Client, cseVersion, templateName string) (string, error) { + return getRemoteFile(client, fmt.Sprintf("https://raw.githubusercontent.com/adambarreiro/go-vcloud-director/new-methods-cse/govcd/cse/%s/%s.tmpl", cseVersion, templateName)) +} + // getRemoteFile gets a Go template file corresponding to the CSE version func getRemoteFile(client *Client, url string) (string, error) { + // resp, err := client.Http.Get(url) if err != nil { return "", err diff --git a/govcd/sample_govcd_test_config.yaml b/govcd/sample_govcd_test_config.yaml index 1af98863b..b4dc10354 100644 --- a/govcd/sample_govcd_test_config.yaml +++ b/govcd/sample_govcd_test_config.yaml @@ -253,3 +253,18 @@ media: nsxtBackedMediaName: nsxtMediaName # A valid UI Plugin to use in tests uiPluginPath: ../test-resources/ui_plugin.zip +cse: + # The organization where Container Service Extension (CSE) Server is running + solutionsOrg: "solutions_org" + # The organization where the Kubernetes clusters are created + tenantOrg: "tenant_org" + # The VDC where the Kubernetes clusters are created + tenantVdc: "tenant_vdc" + # The network which the Kubernetes clusters use + routedNetwork: "tenant_net_routed" + # The edge gateway which the Kubernetes clusters use + edgeGateway: "tenant_edgegateway" + # The catalog which the Kubernetes clusters use + ovaCatalog: "tkgm_catalog" + # The TKGm OVA which the Kubernetes clusters use + ovaName: "ubuntu-2004-kube-v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc" From 301fbc175f679b3d2152bf821a05e55a6be0f126 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 26 Jan 2024 16:31:26 +0100 Subject: [PATCH 005/115] Add delete method Signed-off-by: abarreiro --- govcd/cse.go | 78 ++++++++++++++++++++++++++++++++++++++++++++++- govcd/cse_test.go | 6 ++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/govcd/cse.go b/govcd/cse.go index 499ceb857..7732ba357 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -153,6 +153,82 @@ func (rde *DefinedEntity) CseConvertToCapvcdCluster() (*CseClusterApiProviderClu return result, nil } +// Refresh gets the latest information about the receiver cluster and updates its properties. +func (cluster *CseClusterApiProviderCluster) Refresh() error { + rde, err := getRdeById(cluster.client, cluster.ID) + if err != nil { + return err + } + refreshed, err := rde.CseConvertToCapvcdCluster() + if err != nil { + return err + } + cluster.Capvcd = refreshed.Capvcd + cluster.Etag = refreshed.Etag + return nil +} + +// Delete deletes a CSE Kubernetes cluster, waiting the specified amount of minutes. If the timeout is reached, this method +// returns an error, even if the cluster is already marked for deletion. +func (cluster *CseClusterApiProviderCluster) Delete(timeoutMinutes time.Duration) error { + logHttpResponse := util.LogHttpResponse + + // The following loop is constantly polling VCD to retrieve the RDE, which has a big JSON inside, so we avoid filling + // the log with these big payloads. We use defer to be sure that we restore the initial logging state. + defer func() { + util.LogHttpResponse = logHttpResponse + }() + + var elapsed time.Duration + start := time.Now() + vcdKe := map[string]interface{}{} + for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies timeoutMinutes=0, we wait forever + util.LogHttpResponse = false + rde, err := getRdeById(cluster.client, cluster.ID) + util.LogHttpResponse = logHttpResponse + if err != nil { + if ContainsNotFound(err) { + return nil // The RDE is gone, so the process is completed and there's nothing more to do + } + return fmt.Errorf("could not retrieve the Kubernetes cluster with ID '%s': %s", cluster.ID, err) + } + + spec, ok := rde.DefinedEntity.Entity["spec"].(map[string]interface{}) + if !ok { + return fmt.Errorf("JSON object 'spec' is not correct in the RDE '%s': %s", cluster.ID, err) + } + + vcdKe, ok = spec["vcdKe"].(map[string]interface{}) + if !ok { + return fmt.Errorf("JSON object 'spec.vcdKe' is not correct in the RDE '%s': %s", cluster.ID, err) + } + + if !vcdKe["markForDelete"].(bool) || !vcdKe["forceDelete"].(bool) { + // Mark the cluster for deletion + vcdKe["markForDelete"] = true + vcdKe["forceDelete"] = true + rde.DefinedEntity.Entity["spec"].(map[string]interface{})["vcdKe"] = vcdKe + err = rde.Update(*rde.DefinedEntity) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "etag") { + continue // We ignore any ETag error. This just means a clash with the CSE Server, we just try again + } + return fmt.Errorf("could not mark the Kubernetes cluster with ID '%s' to be deleted: %s", cluster.ID, err) + } + } + + util.Logger.Printf("[DEBUG] Cluster '%s' is still not deleted, will check again in 10 seconds", cluster.ID) + time.Sleep(10 * time.Second) + elapsed = time.Since(start) + } + + // We give a hint to the user about the deletion process result + if len(vcdKe) >= 2 && vcdKe["markForDelete"].(bool) && vcdKe["forceDelete"].(bool) { + return fmt.Errorf("timeout of %v minutes reached, the cluster was successfully marked for deletion but was not removed in time", timeoutMinutes) + } + return fmt.Errorf("timeout of %v minutes reached, the cluster was not marked for deletion, please try again", timeoutMinutes) +} + // waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if timeoutMinutes = 0) // or until this timeout is reached. If the cluster is in "provisioned" state before the given timeout, it returns a CseClusterApiProviderCluster object // representing the Kubernetes cluster with all the latest information. @@ -172,7 +248,7 @@ func waitUntilClusterIsProvisioned(vcdClient *VCDClient, clusterId string, timeo start := time.Now() var capvcdCluster *CseClusterApiProviderCluster - for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies operations_timeout_minutes=0, we wait forever + for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies timeoutMinutes=0, we wait forever util.LogHttpResponse = false rde, err := vcdClient.GetRdeById(clusterId) util.LogHttpResponse = logHttpResponse diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 268a07c6f..3f3782023 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -104,6 +104,12 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(cluster.Etag, Not(Equals), "") check.Assert(cluster.Capvcd.Status.VcdKe.State, Equals, "provisioned") + err = cluster.Refresh() + check.Assert(err, IsNil) + + err = cluster.Delete(0) + check.Assert(err, IsNil) + err = token.Delete() check.Assert(err, IsNil) From 3d6a25551a1301de6b756b1f22b1cc36a2e2405e Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 26 Jan 2024 17:04:35 +0100 Subject: [PATCH 006/115] Init update method Signed-off-by: abarreiro --- govcd/cse.go | 196 ++++++++++++++++++++++++++++++++++------------ govcd/cse_test.go | 8 +- govcd/cse_util.go | 8 +- 3 files changed, 153 insertions(+), 59 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 7732ba357..1aa9a3b28 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -27,23 +27,24 @@ var supportedCseVersions = map[string][]string{ type CseClusterApiProviderCluster struct { Capvcd *types.Capvcd ID string + Owner string Etag string client *Client } -// CseClusterCreationInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// CseClusterCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods // must set in order to create a Kubernetes cluster. -type CseClusterCreationInput struct { +type CseClusterCreateInput struct { Name string OrganizationId string VdcId string NetworkId string KubernetesTemplateOvaId string CseVersion string - ControlPlane ControlPlaneInput - WorkerPools []WorkerPoolInput - DefaultStorageClass *DefaultStorageClassInput // Optional - Owner string // Optional, if not set will pick the current user present in the VCDClient + ControlPlane ControlPlaneCreateInput + WorkerPools []WorkerPoolCreateInput + DefaultStorageClass *DefaultStorageClassCreateInput // Optional + Owner string // Optional, if not set will pick the current user present in the VCDClient ApiToken string NodeHealthCheck bool PodCidr string @@ -53,9 +54,9 @@ type CseClusterCreationInput struct { AutoRepairOnErrors bool } -// ControlPlaneInput defines the required elements that the consumer of these Container Service Extension (CSE) methods -// must set in order to specify the Control Plane inside a CseClusterCreationInput object. -type ControlPlaneInput struct { +// ControlPlaneCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// must set in order to specify the Control Plane inside a CseClusterCreateInput object. +type ControlPlaneCreateInput struct { MachineCount int DiskSizeGi int SizingPolicyId string // Optional @@ -64,9 +65,9 @@ type ControlPlaneInput struct { Ip string // Optional } -// WorkerPoolInput defines the required elements that the consumer of these Container Service Extension (CSE) methods -// must set in order to specify one Worker Pool inside a CseClusterCreationInput object. -type WorkerPoolInput struct { +// WorkerPoolCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// must set in order to specify one Worker Pool inside a CseClusterCreateInput object. +type WorkerPoolCreateInput struct { Name string MachineCount int DiskSizeGi int @@ -76,31 +77,70 @@ type WorkerPoolInput struct { StorageProfileId string // Optional } -// DefaultStorageClassInput defines the required elements that the consumer of these Container Service Extension (CSE) methods -// must set in order to specify a Default Storage Class inside a CseClusterCreationInput object. -type DefaultStorageClassInput struct { +// DefaultStorageClassCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// must set in order to specify a Default Storage Class inside a CseClusterCreateInput object. +type DefaultStorageClassCreateInput struct { StorageProfileId string Name string ReclaimPolicy string Filesystem string } +// CseClusterUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// must set in order to update a Kubernetes cluster. +type CseClusterUpdateInput struct { + KubernetesTemplateOvaId *string + ControlPlane *ControlPlaneUpdateInput + WorkerPools *map[string]WorkerPoolUpdateInput // Maps a node pool name with its contents + NodeHealthCheck *bool + AutoRepairOnErrors *bool +} + +// ControlPlaneUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// must set in order to specify the Control Plane inside a CseClusterUpdateInput object. +type ControlPlaneUpdateInput struct { + MachineCount int +} + +// WorkerPoolUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// must set in order to specify one Worker Pool inside a CseClusterCreateInput object. +type WorkerPoolUpdateInput struct { + MachineCount int +} + //go:embed cse/tkg_versions.json var cseTkgVersionsJson []byte -// CseCreateKubernetesCluster creates a Kubernetes cluster with the data given as input (CseClusterCreationInput). If the given +// CseCreateKubernetesCluster creates a Kubernetes cluster with the data given as input (CseClusterCreateInput). If the given // timeout is 0, it waits forever for the cluster creation. Otherwise, if the timeout is reached and the cluster is not available, // it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster in the returned CseClusterApiProviderCluster. // If the cluster is created correctly, returns all the data in CseClusterApiProviderCluster. -func (vcdClient *VCDClient) CseCreateKubernetesCluster(clusterData CseClusterCreationInput, timeoutMinutes time.Duration) (*CseClusterApiProviderCluster, error) { - goTemplateContents, err := clusterData.toCseClusterCreationGoTemplateContents(vcdClient) +func (vcdClient *VCDClient) CseCreateKubernetesCluster(clusterData CseClusterCreateInput, timeoutMinutes time.Duration) (*CseClusterApiProviderCluster, error) { + clusterId, err := vcdClient.CseCreateKubernetesClusterAsync(clusterData) if err != nil { return nil, err } + cluster, err := waitUntilClusterIsProvisioned(vcdClient, clusterId, timeoutMinutes) + if err != nil { + return cluster, err // Returns the latest status of the cluster + } + + return cluster, nil +} + +// CseCreateKubernetesClusterAsync creates a Kubernetes cluster with the data given as input (CseClusterCreateInput), but does not +// wait for the creation process to finish, so it doesn't monitor for any errors during the process. It returns just the ID of +// the created cluster. One can manually check the status of the cluster with GetKubernetesClusterById and the result of this method. +func (vcdClient *VCDClient) CseCreateKubernetesClusterAsync(clusterData CseClusterCreateInput) (string, error) { + goTemplateContents, err := clusterData.toCseClusterCreationGoTemplateContents(vcdClient) + if err != nil { + return "", err + } + rdeContents, err := getCseKubernetesClusterCreationPayload(vcdClient, goTemplateContents) if err != nil { - return nil, err + return "", err } rde, err := vcdClient.CreateRde("vmware", "capvcdCluster", supportedCseVersions[clusterData.CseVersion][1], types.DefinedEntity{ @@ -112,59 +152,80 @@ func (vcdClient *VCDClient) CseCreateKubernetesCluster(clusterData CseClusterCre OrgName: goTemplateContents.OrganizationName, }) if err != nil { - return nil, err + return "", err } - cluster, err := waitUntilClusterIsProvisioned(vcdClient, rde.DefinedEntity.ID, timeoutMinutes) + return rde.DefinedEntity.ID, nil +} + +// GetKubernetesClusterById retrieves a CSE Kubernetes cluster from VCD by its unique ID +func (vcdClient *VCDClient) GetKubernetesClusterById(id string) (*CseClusterApiProviderCluster, error) { + rde, err := vcdClient.GetRdeById(id) if err != nil { return nil, err } - - return cluster, nil + return rde.cseConvertToCapvcdCluster() } -// CseConvertToCapvcdCluster takes the receiver, which is a generic RDE that must represent an existing CSE Kubernetes cluster, -// and transforms it to a specific Container Service Extension CAPVCD object that represents the same cluster, but -// it is easy to explore and consume. If the receiver object does not contain a CAPVCD object, this method -// will obviously return an error. -func (rde *DefinedEntity) CseConvertToCapvcdCluster() (*CseClusterApiProviderCluster, error) { - requiredType := "vmware:capvcdCluster" - - if !strings.Contains(rde.DefinedEntity.ID, requiredType) || !strings.Contains(rde.DefinedEntity.EntityType, requiredType) { - return nil, fmt.Errorf("the receiver RDE is not a '%s' entity, it is '%s'", requiredType, rde.DefinedEntity.EntityType) +// Refresh gets the latest information about the receiver cluster and updates its properties. +func (cluster *CseClusterApiProviderCluster) Refresh() error { + rde, err := getRdeById(cluster.client, cluster.ID) + if err != nil { + return err } - - entityBytes, err := json.Marshal(rde.DefinedEntity.Entity) + refreshed, err := rde.cseConvertToCapvcdCluster() if err != nil { - return nil, fmt.Errorf("could not marshal the RDE contents to create a Capvcd instance: %s", err) + return err } + cluster.Capvcd = refreshed.Capvcd + cluster.Etag = refreshed.Etag + return nil +} - result := &CseClusterApiProviderCluster{ - Capvcd: &types.Capvcd{}, - ID: rde.DefinedEntity.ID, - Etag: rde.Etag, - client: rde.client, +// Update updates the receiver cluster with the given input. +func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput) error { + if input.NodeHealthCheck != nil { + // TODO + return fmt.Errorf("not implemented") } + if input.AutoRepairOnErrors != nil { + cluster.Capvcd.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors + } + if input.KubernetesTemplateOvaId != nil { + // TODO: Get YAML, search for machines, change templateName + return fmt.Errorf("not implemented") + } + if input.ControlPlane != nil { + // TODO: Get YAML, search for control plane, change replicas + return fmt.Errorf("not implemented") + } + if input.WorkerPools != nil { + for name, updateDetails := range *input.WorkerPools { + // TODO: Get YAML, search for node pool with name == 'name', if matches, change replicas + return fmt.Errorf("not implemented %s, %v", name, updateDetails) + } - err = json.Unmarshal(entityBytes, result.Capvcd) - if err != nil { - return nil, fmt.Errorf("could not unmarshal the RDE contents to create a Capvcd instance: %s", err) + return fmt.Errorf("not implemented") } - return result, nil -} -// Refresh gets the latest information about the receiver cluster and updates its properties. -func (cluster *CseClusterApiProviderCluster) Refresh() error { + marshaledPayload, err := json.Marshal(cluster.Capvcd) + if err != nil { + return err + } + entityContent := map[string]interface{}{} + err = json.Unmarshal(marshaledPayload, &entityContent) + if err != nil { + return err + } rde, err := getRdeById(cluster.client, cluster.ID) if err != nil { return err } - refreshed, err := rde.CseConvertToCapvcdCluster() + rde.DefinedEntity.Entity = entityContent + err = rde.Update(*rde.DefinedEntity) if err != nil { return err } - cluster.Capvcd = refreshed.Capvcd - cluster.Etag = refreshed.Etag return nil } @@ -229,6 +290,39 @@ func (cluster *CseClusterApiProviderCluster) Delete(timeoutMinutes time.Duration return fmt.Errorf("timeout of %v minutes reached, the cluster was not marked for deletion, please try again", timeoutMinutes) } +// cseConvertToCapvcdCluster takes the receiver, which is a generic RDE that must represent an existing CSE Kubernetes cluster, +// and transforms it to a specific Container Service Extension CAPVCD object that represents the same cluster, but +// it is easy to explore and consume. If the receiver object does not contain a CAPVCD object, this method +// will obviously return an error. +func (rde *DefinedEntity) cseConvertToCapvcdCluster() (*CseClusterApiProviderCluster, error) { + requiredType := "vmware:capvcdCluster" + + if !strings.Contains(rde.DefinedEntity.ID, requiredType) || !strings.Contains(rde.DefinedEntity.EntityType, requiredType) { + return nil, fmt.Errorf("the receiver RDE is not a '%s' entity, it is '%s'", requiredType, rde.DefinedEntity.EntityType) + } + + entityBytes, err := json.Marshal(rde.DefinedEntity.Entity) + if err != nil { + return nil, fmt.Errorf("could not marshal the RDE contents to create a Capvcd instance: %s", err) + } + + result := &CseClusterApiProviderCluster{ + Capvcd: &types.Capvcd{}, + ID: rde.DefinedEntity.ID, + Etag: rde.Etag, + client: rde.client, + } + if rde.DefinedEntity.Owner != nil { + result.Owner = rde.DefinedEntity.Owner.Name + } + + err = json.Unmarshal(entityBytes, result.Capvcd) + if err != nil { + return nil, fmt.Errorf("could not unmarshal the RDE contents to create a Capvcd instance: %s", err) + } + return result, nil +} + // waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if timeoutMinutes = 0) // or until this timeout is reached. If the cluster is in "provisioned" state before the given timeout, it returns a CseClusterApiProviderCluster object // representing the Kubernetes cluster with all the latest information. @@ -256,7 +350,7 @@ func waitUntilClusterIsProvisioned(vcdClient *VCDClient, clusterId string, timeo return nil, err } - capvcdCluster, err = rde.CseConvertToCapvcdCluster() + capvcdCluster, err = rde.cseConvertToCapvcdCluster() if err != nil { return nil, err } diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 3f3782023..14497fba2 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -65,28 +65,28 @@ func (vcd *TestVCD) Test_Cse(check *C) { apiToken, err := token.GetInitialApiToken() check.Assert(err, IsNil) - cluster, err := vcd.client.CseCreateKubernetesCluster(CseClusterCreationInput{ + cluster, err := vcd.client.CseCreateKubernetesCluster(CseClusterCreateInput{ Name: "test-cse", OrganizationId: org.Org.ID, VdcId: vdc.Vdc.ID, NetworkId: net.OrgVDCNetwork.ID, KubernetesTemplateOvaId: ova.VAppTemplate.ID, CseVersion: "4.2", - ControlPlane: ControlPlaneInput{ + ControlPlane: ControlPlaneCreateInput{ MachineCount: 1, DiskSizeGi: 20, SizingPolicyId: policies[0].VdcComputePolicyV2.ID, StorageProfileId: sp.ID, Ip: "", }, - WorkerPools: []WorkerPoolInput{{ + WorkerPools: []WorkerPoolCreateInput{{ Name: "worker-pool-1", MachineCount: 1, DiskSizeGi: 20, SizingPolicyId: policies[0].VdcComputePolicyV2.ID, StorageProfileId: sp.ID, }}, - DefaultStorageClass: &DefaultStorageClassInput{ + DefaultStorageClass: &DefaultStorageClassCreateInput{ StorageProfileId: sp.ID, Name: "storage-class-1", ReclaimPolicy: "delete", diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 8644c0f6f..0a6e96c00 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -14,7 +14,7 @@ import ( // cseClusterCreationGoTemplateArguments defines the required arguments that are required by the Go templates used internally to specify // a Kubernetes cluster. These are not set by the user, but instead they are computed from a valid -// CseClusterCreationInput object in the CseClusterCreationInput.toCseClusterCreationGoTemplateContents method. These fields are then +// CseClusterCreateInput object in the CseClusterCreateInput.toCseClusterCreationGoTemplateContents method. These fields are then // inserted in Go templates to render a final JSON that is valid to be used as the cluster Runtime Defined Entity (RDE) payload. type cseClusterCreationGoTemplateArguments struct { CseVersion string @@ -81,7 +81,7 @@ type machineHealthCheck struct { } // validate validates the CSE Kubernetes cluster creation input data. Returns an error if some of the fields is wrong. -func (ccd *CseClusterCreationInput) validate() error { +func (ccd *CseClusterCreateInput) validate() error { cseNamesRegex, err := regexp.Compile(`^[a-z](?:[a-z0-9-]{0,29}[a-z0-9])?$`) if err != nil { return fmt.Errorf("could not compile regular expression '%s'", err) @@ -153,9 +153,9 @@ func (ccd *CseClusterCreationInput) validate() error { return nil } -// toCseClusterCreationGoTemplateContents transforms user input data (receiver CseClusterCreationInput) into the final payload that +// toCseClusterCreationGoTemplateContents transforms user input data (receiver CseClusterCreateInput) into the final payload that // will be used to render the Go templates that define a Kubernetes cluster creation payload (cseClusterCreationGoTemplateArguments). -func (input *CseClusterCreationInput) toCseClusterCreationGoTemplateContents(vcdClient *VCDClient) (*cseClusterCreationGoTemplateArguments, error) { +func (input *CseClusterCreateInput) toCseClusterCreationGoTemplateContents(vcdClient *VCDClient) (*cseClusterCreationGoTemplateArguments, error) { err := input.validate() if err != nil { return nil, err From 60d259bb57877416e6a9eb4f2ed80d47921d19aa Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 30 Jan 2024 13:24:25 +0100 Subject: [PATCH 007/115] Fix issues and run tests Signed-off-by: abarreiro --- govcd/cse.go | 6 ++-- govcd/cse_template.go | 8 ++--- govcd/cse_test.go | 17 +++++------ govcd/cse_util.go | 68 +++++++++++++++++-------------------------- 4 files changed, 42 insertions(+), 57 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 1aa9a3b28..2a5396044 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -1,7 +1,7 @@ package govcd import ( - _ "embed" + "embed" "encoding/json" "fmt" "github.com/vmware/go-vcloud-director/v2/types/v56" @@ -108,8 +108,8 @@ type WorkerPoolUpdateInput struct { MachineCount int } -//go:embed cse/tkg_versions.json -var cseTkgVersionsJson []byte +//go:embed cse +var cseFiles embed.FS // CseCreateKubernetesCluster creates a Kubernetes cluster with the data given as input (CseClusterCreateInput). If the given // timeout is 0, it waits forever for the cluster creation. Otherwise, if the timeout is reached and the cluster is not available, diff --git a/govcd/cse_template.go b/govcd/cse_template.go index a0342cde6..465ca42a6 100644 --- a/govcd/cse_template.go +++ b/govcd/cse_template.go @@ -39,7 +39,7 @@ func getCseKubernetesClusterCreationPayload(vcdClient *VCDClient, goTemplateCont args["DefaultStorageClassFileSystem"] = goTemplateContents.DefaultStorageClass.Filesystem } - rdeTmpl, err := getCseTemplate(&vcdClient.Client, goTemplateContents.CseVersion, "rde") + rdeTmpl, err := getCseTemplate(goTemplateContents.CseVersion, "rde") if err != nil { return nil, err } @@ -61,7 +61,7 @@ func getCseKubernetesClusterCreationPayload(vcdClient *VCDClient, goTemplateCont // generateNodePoolYaml generates YAML blocks corresponding to the Kubernetes node pools. func generateNodePoolYaml(vcdClient *VCDClient, clusterDetails *cseClusterCreationGoTemplateArguments) (string, error) { - workerPoolTmpl, err := getCseTemplate(&vcdClient.Client, clusterDetails.CseVersion, "capiyaml_workerpool") + workerPoolTmpl, err := getCseTemplate(clusterDetails.CseVersion, "capiyaml_workerpool") if err != nil { return "", err } @@ -110,7 +110,7 @@ func generateMemoryHealthCheckYaml(vcdClient *VCDClient, clusterDetails *cseClus return "", nil } - mhcTmpl, err := getCseTemplate(&vcdClient.Client, clusterDetails.CseVersion, "capiyaml_mhc") + mhcTmpl, err := getCseTemplate(clusterDetails.CseVersion, "capiyaml_mhc") if err != nil { return "", err } @@ -136,7 +136,7 @@ func generateMemoryHealthCheckYaml(vcdClient *VCDClient, clusterDetails *cseClus // in the CAPVCD cluster JSON payload. This function picks data from the Terraform schema and the createClusterDto to // populate several Go templates and build a final YAML. func generateCapiYaml(vcdClient *VCDClient, clusterDetails *cseClusterCreationGoTemplateArguments) (string, error) { - clusterTmpl, err := getCseTemplate(&vcdClient.Client, clusterDetails.CseVersion, "capiyaml_cluster") + clusterTmpl, err := getCseTemplate(clusterDetails.CseVersion, "capiyaml_cluster") if err != nil { return "", err } diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 14497fba2..544aeef7a 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -29,10 +29,6 @@ func skipCseTests(testConfig TestConfig) bool { // Test_Cse func (vcd *TestVCD) Test_Cse(check *C) { - if vcd.skipAdminTests { - check.Skip(fmt.Sprintf(TestRequiresSysAdminPrivileges, check.TestName())) - } - if skipCseTests(vcd.config) { check.Skip(fmt.Sprintf(TestRequiresCseConfiguration, check.TestName())) } @@ -40,25 +36,28 @@ func (vcd *TestVCD) Test_Cse(check *C) { org, err := vcd.client.GetOrgByName(vcd.config.Cse.TenantOrg) check.Assert(err, IsNil) - vdc, err := org.GetVDCByName(vcd.config.Cse.TenantOrg, false) + catalog, err := org.GetCatalogByName(vcd.config.Cse.OvaCatalog, false) check.Assert(err, IsNil) - net, err := vdc.GetOrgVdcNetworkByName(vcd.config.Cse.RoutedNetwork, false) + ova, err := catalog.GetVAppTemplateByName(vcd.config.Cse.OvaName) check.Assert(err, IsNil) - ova, err := vdc.GetVAppTemplateByName(vcd.config.Cse.OvaName) + vdc, err := org.GetVDCByName(vcd.config.Cse.TenantVdc, false) + check.Assert(err, IsNil) + + net, err := vdc.GetOrgVdcNetworkByName(vcd.config.Cse.RoutedNetwork, false) check.Assert(err, IsNil) sp, err := vdc.FindStorageProfileReference(vcd.config.VCD.StorageProfile.SP1) check.Assert(err, IsNil) policies, err := vcd.client.GetAllVdcComputePoliciesV2(url.Values{ - "filter": []string{"name==*TKG%20small*"}, + "filter": []string{"name==TKG small"}, }) check.Assert(err, IsNil) check.Assert(len(policies), Equals, 1) - token, err := vcd.client.CreateToken(vcd.config.Provider.SysOrg, check.TestName()) + token, err := vcd.client.CreateToken(vcd.config.Provider.SysOrg, check.TestName()+"123") // TODO: Remove 123 check.Assert(err, IsNil) AddToCleanupListOpenApi(token.Token.Name, check.TestName(), types.OpenApiPathVersion1_0_0+types.OpenApiEndpointTokens+token.Token.ID) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 0a6e96c00..0d18b5e5b 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -5,9 +5,6 @@ import ( "encoding/json" "fmt" "github.com/vmware/go-vcloud-director/v2/types/v56" - "github.com/vmware/go-vcloud-director/v2/util" - "io" - "net/http" "regexp" "strings" ) @@ -299,12 +296,15 @@ func (input *CseClusterCreateInput) toCseClusterCreationGoTemplateContents(vcdCl output.VcdUrl = strings.Replace(vcdClient.Client.VCDHREF.String(), "/api", "", 1) // These don't change, don't need mapping + output.ApiToken = input.ApiToken + output.AutoRepairOnErrors = input.AutoRepairOnErrors + output.CseVersion = input.CseVersion + output.Name = input.Name output.PodCidr = input.PodCidr output.ServiceCidr = input.ServiceCidr output.SshPublicKey = input.SshPublicKey output.VirtualIpSubnet = input.VirtualIpSubnet - output.AutoRepairOnErrors = input.AutoRepairOnErrors - output.CseVersion = input.CseVersion + return output, nil } @@ -336,8 +336,13 @@ func getTkgVersionBundleFromVAppTemplateName(ovaName string) (tkgVersionBundle, } parsedOvaName := strings.ReplaceAll(ovaName, ".ova", "")[cutPosition+len("kube-"):] + cseTkgVersionsJson, err := cseFiles.ReadFile("cse/tkg_versions.json") + if err != nil { + return result, err + } + versionsMap := map[string]interface{}{} - err := json.Unmarshal(cseTkgVersionsJson, &versionsMap) + err = json.Unmarshal(cseTkgVersionsJson, &versionsMap) if err != nil { return result, err } @@ -357,10 +362,12 @@ func getTkgVersionBundleFromVAppTemplateName(ovaName string) (tkgVersionBundle, // getMachineHealthCheck gets the required information from the CSE Server configuration RDE func getMachineHealthCheck(vcdClient *VCDClient, cseVersion string, isNodeHealthCheckActive bool) (*machineHealthCheck, error) { + if !isNodeHealthCheckActive { + return nil, nil + } currentCseVersion := supportedCseVersions[cseVersion] - result := machineHealthCheck{} - rdes, err := vcdClient.GetRdesByName("vmware", "VCDKEConfig", currentCseVersion[0], "machineHealthCheck") + rdes, err := vcdClient.GetRdesByName("vmware", "VCDKEConfig", currentCseVersion[0], "vcdKeConfig") if err != nil { return nil, fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", currentCseVersion[0], err) } @@ -376,22 +383,21 @@ func getMachineHealthCheck(vcdClient *VCDClient, cseVersion string, isNodeHealth return nil, fmt.Errorf("wrong format of VCDKEConfig, expected a single 'profiles' element, got %d", len(profiles)) } - if isNodeHealthCheckActive { - // TODO: Get the struct Type for this one - mhc := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"].(map[string]interface{}) - result.MaxUnhealthyNodesPercentage = mhc["maxUnhealthyNodes"].(float64) - result.NodeStartupTimeout = mhc["nodeStartupTimeout"].(string) - result.NodeNotReadyTimeout = mhc["nodeUnknownTimeout"].(string) - result.NodeUnknownTimeout = mhc["nodeNotReadyTimeout"].(string) - } - return nil, nil + // TODO: Get the struct Type for this one + result := machineHealthCheck{} + mhc := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"].(map[string]interface{}) + result.MaxUnhealthyNodesPercentage = mhc["maxUnhealthyNodes"].(float64) + result.NodeStartupTimeout = mhc["nodeStartupTimeout"].(string) + result.NodeNotReadyTimeout = mhc["nodeUnknownTimeout"].(string) + result.NodeUnknownTimeout = mhc["nodeNotReadyTimeout"].(string) + return &result, nil } // getContainerRegistryUrl gets the required information from the CSE Server configuration RDE func getContainerRegistryUrl(vcdClient *VCDClient, cseVersion string) (string, error) { currentCseVersion := supportedCseVersions[cseVersion] - rdes, err := vcdClient.GetRdesByName("vmware", "VCDKEConfig", currentCseVersion[0], "machineHealthCheck") + rdes, err := vcdClient.GetRdesByName("vmware", "VCDKEConfig", currentCseVersion[0], "vcdKeConfig") if err != nil { return "", fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", currentCseVersion[0], err) } @@ -410,32 +416,12 @@ func getContainerRegistryUrl(vcdClient *VCDClient, cseVersion string) (string, e return fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"].(string)), nil } -func getCseTemplate(client *Client, cseVersion, templateName string) (string, error) { - return getRemoteFile(client, fmt.Sprintf("https://raw.githubusercontent.com/adambarreiro/go-vcloud-director/new-methods-cse/govcd/cse/%s/%s.tmpl", cseVersion, templateName)) -} - -// getRemoteFile gets a Go template file corresponding to the CSE version -func getRemoteFile(client *Client, url string) (string, error) { - // - resp, err := client.Http.Get(url) +func getCseTemplate(cseVersion, templateName string) (string, error) { + result, err := cseFiles.ReadFile(fmt.Sprintf("cse/%s/%s.tmpl", cseVersion, templateName)) if err != nil { return "", err } - defer func() { - if err := resp.Body.Close(); err != nil { - util.Logger.Printf("[ERROR] getRemoteFile: Could not close HTTP response body: %s", err) - } - }() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("could not get file from URL %s, got status %s", url, resp.Status) - } - - response, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - return string(response), nil + return string(result), nil } // getKeys retrieves all the keys from the given map and returns them as a slice From ca58e692b69754e3791634f6bbf049d265263e34 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 31 Jan 2024 13:00:04 +0100 Subject: [PATCH 008/115] Add unit tests Signed-off-by: abarreiro --- govcd/cse.go | 113 ++++++---- govcd/cse_test.go | 33 ++- govcd/cse_yaml.go | 288 ++++++++++++++++++++++++++ govcd/cse_yaml_unit_test.go | 321 +++++++++++++++++++++++++++++ govcd/test-resources/capiYaml.yaml | 210 +++++++++++++++++++ 5 files changed, 918 insertions(+), 47 deletions(-) create mode 100644 govcd/cse_yaml.go create mode 100644 govcd/cse_yaml_unit_test.go create mode 100644 govcd/test-resources/capiYaml.yaml diff --git a/govcd/cse.go b/govcd/cse.go index 3f35c0b7d..66aa5ca30 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -41,10 +41,10 @@ type CseClusterCreateInput struct { NetworkId string KubernetesTemplateOvaId string CseVersion string - ControlPlane ControlPlaneCreateInput - WorkerPools []WorkerPoolCreateInput - DefaultStorageClass *DefaultStorageClassCreateInput // Optional - Owner string // Optional, if not set will pick the current user present in the VCDClient + ControlPlane CseControlPlaneCreateInput + WorkerPools []CseWorkerPoolCreateInput + DefaultStorageClass *CseDefaultStorageClassCreateInput // Optional + Owner string // Optional, if not set will pick the current user present in the VCDClient ApiToken string NodeHealthCheck bool PodCidr string @@ -54,9 +54,9 @@ type CseClusterCreateInput struct { AutoRepairOnErrors bool } -// ControlPlaneCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// CseControlPlaneCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods // must set in order to specify the Control Plane inside a CseClusterCreateInput object. -type ControlPlaneCreateInput struct { +type CseControlPlaneCreateInput struct { MachineCount int DiskSizeGi int SizingPolicyId string // Optional @@ -65,9 +65,9 @@ type ControlPlaneCreateInput struct { Ip string // Optional } -// WorkerPoolCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// CseWorkerPoolCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods // must set in order to specify one Worker Pool inside a CseClusterCreateInput object. -type WorkerPoolCreateInput struct { +type CseWorkerPoolCreateInput struct { Name string MachineCount int DiskSizeGi int @@ -77,9 +77,9 @@ type WorkerPoolCreateInput struct { StorageProfileId string // Optional } -// DefaultStorageClassCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// CseDefaultStorageClassCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods // must set in order to specify a Default Storage Class inside a CseClusterCreateInput object. -type DefaultStorageClassCreateInput struct { +type CseDefaultStorageClassCreateInput struct { StorageProfileId string Name string ReclaimPolicy string @@ -90,21 +90,21 @@ type DefaultStorageClassCreateInput struct { // must set in order to update a Kubernetes cluster. type CseClusterUpdateInput struct { KubernetesTemplateOvaId *string - ControlPlane *ControlPlaneUpdateInput - WorkerPools *map[string]WorkerPoolUpdateInput // Maps a node pool name with its contents + ControlPlane *CseControlPlaneUpdateInput + WorkerPools *map[string]CseWorkerPoolUpdateInput // Maps a node pool name with its contents NodeHealthCheck *bool AutoRepairOnErrors *bool } -// ControlPlaneUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// CseControlPlaneUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods // must set in order to specify the Control Plane inside a CseClusterUpdateInput object. -type ControlPlaneUpdateInput struct { +type CseControlPlaneUpdateInput struct { MachineCount int } -// WorkerPoolUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods +// CseWorkerPoolUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods // must set in order to specify one Worker Pool inside a CseClusterCreateInput object. -type WorkerPoolUpdateInput struct { +type CseWorkerPoolUpdateInput struct { MachineCount int } @@ -190,7 +190,7 @@ func (cluster *CseClusterApiProviderCluster) Refresh() error { // timeout is 0, it waits forever for the cluster update to finish. Otherwise, if the timeout is reached and the cluster is not available, // it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster will be available in the // receiver CseClusterApiProviderCluster. -func (cluster *CseClusterApiProviderCluster) UpdateWorkerPools(input map[string]WorkerPoolUpdateInput, timeoutMinutes time.Duration) error { +func (cluster *CseClusterApiProviderCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput, timeoutMinutes time.Duration) error { return cluster.Update(CseClusterUpdateInput{ WorkerPools: &input, }, timeoutMinutes) @@ -200,7 +200,7 @@ func (cluster *CseClusterApiProviderCluster) UpdateWorkerPools(input map[string] // timeout is 0, it waits forever for the cluster update to finish. Otherwise, if the timeout is reached and the cluster is not available, // it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster will be available in the // receiver CseClusterApiProviderCluster. -func (cluster *CseClusterApiProviderCluster) UpdateControlPlane(input ControlPlaneUpdateInput, timeoutMinutes time.Duration) error { +func (cluster *CseClusterApiProviderCluster) UpdateControlPlane(input CseControlPlaneUpdateInput, timeoutMinutes time.Duration) error { return cluster.Update(CseClusterUpdateInput{ ControlPlane: &input, }, timeoutMinutes) @@ -239,29 +239,25 @@ func (cluster *CseClusterApiProviderCluster) SetAutoRepairOnErrors(autoRepairOnE // it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster will be available in the // receiver CseClusterApiProviderCluster. func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput, timeoutMinutes time.Duration) error { - if input.NodeHealthCheck != nil { - // TODO - return fmt.Errorf("not implemented") - } - if input.AutoRepairOnErrors != nil { - cluster.Capvcd.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors + err := cluster.Refresh() + if err != nil { + return err } - if input.KubernetesTemplateOvaId != nil { - // TODO: Get YAML, search for machines, change templateName - return fmt.Errorf("not implemented") + if cluster.Capvcd.Status.VcdKe.State == "" { + return fmt.Errorf("can't update a Kubernetes cluster that does not have any state") } - if input.ControlPlane != nil { - // TODO: Get YAML, search for control plane, change replicas - return fmt.Errorf("not implemented") + if cluster.Capvcd.Status.VcdKe.State != "provisioned" { + return fmt.Errorf("can't update a Kubernetes cluster that is not in 'provisioned' state, as it is in '%s'", cluster.Capvcd.Status.VcdKe.State) } - if input.WorkerPools != nil { - for name, updateDetails := range *input.WorkerPools { - // TODO: Get YAML, search for node pool with name == 'name', if matches, change replicas - return fmt.Errorf("not implemented %s, %v", name, updateDetails) - } - return fmt.Errorf("not implemented") + if input.AutoRepairOnErrors != nil { + cluster.Capvcd.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors + } + updatedCapiYaml, err := cseUpdateCapiYaml(cluster.Capvcd.Spec.CapiYaml, input) + if err != nil { + return err } + cluster.Capvcd.Spec.CapiYaml = updatedCapiYaml marshaledPayload, err := json.Marshal(cluster.Capvcd) if err != nil { @@ -272,14 +268,45 @@ func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput, if err != nil { return err } - rde, err := getRdeById(cluster.client, cluster.ID) - if err != nil { - return err + + logHttpResponse := util.LogHttpResponse + // The following loop is constantly polling VCD to retrieve the RDE, which has a big JSON inside, so we avoid filling + // the log with these big payloads. We use defer to be sure that we restore the initial logging state. + defer func() { + util.LogHttpResponse = logHttpResponse + }() + + // We do this loop to increase the chances that the Kubernetes cluster is successfully created, as the Go SDK is + // "fighting" with the CSE Server + retries := 0 + maxRetries := 5 + updated := false + for retries <= maxRetries { + util.LogHttpResponse = false + rde, err := getRdeById(cluster.client, cluster.ID) + util.LogHttpResponse = logHttpResponse + if err != nil { + return err + } + + rde.DefinedEntity.Entity = entityContent + err = rde.Update(*rde.DefinedEntity) + if err == nil { + updated = true + break + } + if err != nil { + // If it's an ETag error, we just retry + if !strings.Contains(strings.ToLower(err.Error()), "etag") { + return err + } + } + retries++ + util.Logger.Printf("[DEBUG] The request to update the Kubernetes cluster '%s' failed due to a ETag lock. Trying again", cluster.ID) } - rde.DefinedEntity.Entity = entityContent - err = rde.Update(*rde.DefinedEntity) - if err != nil { - return err + + if !updated { + return fmt.Errorf("could not update the Kubernetes cluster '%s' due to %d ETag locks that blocked the operation", cluster.ID, maxRetries) } _, finalError := waitUntilClusterIsProvisioned(cluster.client, cluster.ID, timeoutMinutes) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 7b1c54d3b..51293360e 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -73,21 +73,21 @@ func (vcd *TestVCD) Test_Cse(check *C) { NetworkId: net.OrgVDCNetwork.ID, KubernetesTemplateOvaId: ova.VAppTemplate.ID, CseVersion: "4.2", - ControlPlane: ControlPlaneCreateInput{ + ControlPlane: CseControlPlaneCreateInput{ MachineCount: 1, DiskSizeGi: 20, SizingPolicyId: policies[0].VdcComputePolicyV2.ID, StorageProfileId: sp.ID, Ip: "", }, - WorkerPools: []WorkerPoolCreateInput{{ + WorkerPools: []CseWorkerPoolCreateInput{{ Name: workerPoolName, MachineCount: 1, DiskSizeGi: 20, SizingPolicyId: policies[0].VdcComputePolicyV2.ID, StorageProfileId: sp.ID, }}, - DefaultStorageClass: &DefaultStorageClassCreateInput{ + DefaultStorageClass: &CseDefaultStorageClassCreateInput{ StorageProfileId: sp.ID, Name: "storage-class-1", ReclaimPolicy: "delete", @@ -124,7 +124,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { } } // Perform the update - err = cluster.UpdateWorkerPools(map[string]WorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, 0) + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, 0) check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up @@ -144,3 +144,28 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(err, IsNil) } + +func (vcd *TestVCD) Test_Deleteme(check *C) { + org, err := vcd.client.GetOrgByName(vcd.config.Cse.TenantOrg) + check.Assert(err, IsNil) + + cluster, err := org.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:e8e82bcc-50a1-484f-9dd0-20965ab3e865") + check.Assert(err, IsNil) + + workerPoolName := "worker-node-pool-1" + + // Perform the update + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, 0) + check.Assert(err, IsNil) + + // Post-check. This should be 2, as it should have scaled up + foundWorkerPool := false + for _, nodePool := range cluster.Capvcd.Status.Capvcd.NodePool { + if nodePool.Name == workerPoolName { + foundWorkerPool = true + check.Assert(nodePool.DesiredReplicas, Equals, 2) + } + } + check.Assert(foundWorkerPool, Equals, true) + +} diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go new file mode 100644 index 000000000..145c0967f --- /dev/null +++ b/govcd/cse_yaml.go @@ -0,0 +1,288 @@ +package govcd + +import ( + "bytes" + "fmt" + "gopkg.in/yaml.v2" + "strings" +) + +// traverseMapAndGet traverses the input interface{}, which should be a map of maps, by following the path specified as +// "keyA.keyB.keyC.keyD", doing something similar to, visually speaking, map["keyA"]["keyB"]["keyC"]["keyD"], or in other words, +// it goes inside every inner map, which are inside the initial map, until the given path is finished. +// The final value, "keyD" in the same example, should be of type ResultType, which is a generic type requested during the call +// to this function. +func traverseMapAndGet[ResultType any](input interface{}, path string) (ResultType, error) { + var nothing ResultType + if input == nil { + return nothing, fmt.Errorf("the input is nil") + } + inputMap, ok := input.(map[any]any) + if !ok { + return nothing, fmt.Errorf("the input is a %T, not a map[string]interface{}", input) + } + if len(inputMap) == 0 { + return nothing, fmt.Errorf("the map is empty") + } + pathUnits := strings.Split(path, ".") + completed := false + i := 0 + var result interface{} + for !completed { + subPath := pathUnits[i] + traversed, ok := inputMap[subPath] + if !ok { + return nothing, fmt.Errorf("key '%s' does not exist in input map", subPath) + } + if i < len(pathUnits)-1 { + traversedMap, ok := traversed.(map[any]any) + if !ok { + return nothing, fmt.Errorf("key '%s' is a %T, not a map[string]interface{}, but there are still %d paths to explore", subPath, traversed, len(pathUnits)-(i+1)) + } + inputMap = traversedMap + } else { + completed = true + result = traversed + } + i++ + } + resultTyped, ok := result.(ResultType) + if !ok { + return nothing, fmt.Errorf("could not convert obtained type %T to requested %T", result, nothing) + } + return resultTyped, nil +} + +func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[any]any, kubernetesTemplateOvaName string) error { + updated := false + for _, d := range yamlDocuments { + if d["kind"] != "VCDMachineTemplate" { + continue + } + // Check that it is a control plane pool by checking the name + name, err := traverseMapAndGet[string](d, "metadata.name") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + if !strings.Contains(name, "control-plane-node-pool") { + continue + } + + // Perform the update + _, err = traverseMapAndGet[string](d, "spec.template.spec.template") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + d["spec"].(map[any]any)["template"].(map[any]any)["spec"].(map[any]any)["template"] = kubernetesTemplateOvaName + updated = true + } + if !updated { + return fmt.Errorf("could not find any Control Plane pool in the CAPI YAML") + } + return nil +} + +func updateControlPlaneYaml(docs []map[any]any, input CseControlPlaneUpdateInput) error { + return nil +} + +func updateWorkerPoolsYaml(docs []map[any]any, m map[string]CseWorkerPoolUpdateInput) error { + return nil +} + +func updateNodeHealthCheckYaml(docs []map[any]any, b bool) error { + return nil +} + +// cseUpdateCapiYaml takes a CAPI YAML and modifies its Kubernetes template, its Control plane, its Worker pools +// and its Node Health Check capabilities, by using the new values provided as input. +// If some of the values of the input is not provided, it doesn't change them. +// If none of the values is provided, it just returns the same untouched YAML. +func cseUpdateCapiYaml(capiYaml string, input CseClusterUpdateInput) (string, error) { + if input.ControlPlane == nil && input.WorkerPools == nil && input.NodeHealthCheck == nil && input.KubernetesTemplateOvaId == nil { + return capiYaml, nil + } + + // The CAPI YAML contains multiple documents, so we cannot use a simple yaml.Unmarshal() as this one just gets the first + // document it finds. + yamlDocs, err := unmarshalMultipleYamlDocuments(capiYaml) + if err != nil { + return "", fmt.Errorf("error unmarshaling CAPI YAML: %s", err) + } + + // As a side note, we can't optimize this one with a 'if currentValue equals updatedValue do nothing' because + // in order to retrieve the current value we would need to explore the YAML anyway, which is what we also need to do to update it. + // So in this special case this "optimization" would optimize nothing. The same happens with other YAML values. + if input.KubernetesTemplateOvaId != nil { + + err := cseUpdateKubernetesTemplateInYaml(yamlDocs, *input.KubernetesTemplateOvaId) + if err != nil { + return "", err + } + } + + if input.ControlPlane != nil { + err := updateControlPlaneYaml(yamlDocs, *input.ControlPlane) + if err != nil { + return "", err + } + } + + if input.WorkerPools != nil { + err := updateWorkerPoolsYaml(yamlDocs, *input.WorkerPools) + if err != nil { + return "", err + } + } + + if input.NodeHealthCheck != nil { + err := updateNodeHealthCheckYaml(yamlDocs, *input.NodeHealthCheck) + if err != nil { + return "", err + } + } + + return marshalMultipleYamlDocuments(yamlDocs) + /* + if d.HasChange("control_plane.0.machine_count") { + for _, yamlDoc := range yamlDocs { + if yamlDoc["kind"] == "KubeadmControlPlane" { + yamlDoc["spec"].(map[string]interface{})["replicas"] = d.Get("control_plane.0.machine_count") + } + } + } + // The node pools can only be created and resized + var newNodePools []map[string]interface{} + if d.HasChange("node_pool") { + for _, nodePoolRaw := range d.Get("node_pool").(*schema.Set).List() { + nodePool := nodePoolRaw.(map[string]interface{}) + for _, yamlDoc := range yamlDocs { + if yamlDoc["kind"] == "MachineDeployment" { + if yamlDoc["metadata"].(map[string]interface{})["name"] == nodePool["name"].(string) { + yamlDoc["spec"].(map[string]interface{})["replicas"] = nodePool["machine_count"].(int) + } else { + // TODO: Create node pool + newNodePools = append(newNodePools, map[string]interface{}{}) + } + } + } + } + } + if len(newNodePools) > 0 { + yamlDocs = append(yamlDocs, newNodePools...) + } + + if d.HasChange("node_health_check") { + oldNhc, newNhc := d.GetChange("node_health_check") + if oldNhc.(bool) && !newNhc.(bool) { + toDelete := 0 + for i, yamlDoc := range yamlDocs { + if yamlDoc["kind"] == "MachineHealthCheck" { + toDelete = i + } + } + yamlDocs[toDelete] = yamlDocs[len(yamlDocs)-1] // We delete the MachineHealthCheck block by putting the last doc in its place + yamlDocs = yamlDocs[:len(yamlDocs)-1] // Then we remove the last doc + } else { + // Add the YAML block + vcdKeConfig, err := getVcdKeConfiguration(d, vcdClient) + if err != nil { + return diag.FromErr(err) + } + rawYaml, err := generateMemoryHealthCheckYaml(d, vcdClient, *vcdKeConfig, d.Get("name").(string)) + if err != nil { + return diag.FromErr(err) + } + yamlBlock := map[string]interface{}{} + err = yaml.Unmarshal([]byte(rawYaml), &yamlBlock) + if err != nil { + return diag.Errorf("error updating Memory Health Check: %s", err) + } + yamlDocs = append(yamlDocs, yamlBlock) + } + util.Logger.Printf("not done but make static complains :)") + } + + updatedYaml, err := yaml.Marshal(yamlDocs) + if err != nil { + return diag.Errorf("error updating cluster: %s", err) + } + + // This must be done with retries due to the possible clash on ETags + _, err = runWithRetry( + "update cluster", + "could not update cluster", + 1*time.Minute, + nil, + func() (any, error) { + rde, err := vcdClient.GetRdeById(d.Id()) + if err != nil { + return nil, fmt.Errorf("could not update Kubernetes cluster with ID '%s': %s", d.Id(), err) + } + + rde.DefinedEntity.Entity["spec"].(map[string]interface{})["capiYaml"] = updatedYaml + rde.DefinedEntity.Entity["spec"].(map[string]interface{})["vcdKe"].(map[string]interface{})["autoRepairOnErrors"] = d.Get("auto_repair_on_errors").(bool) + + // err = rde.Update(*rde.DefinedEntity) + util.Logger.Printf("ADAM: PERFORM UPDATE: %v", rde.DefinedEntity.Entity) + if err != nil { + return nil, err + } + return nil, nil + }, + ) + if err != nil { + return diag.FromErr(err) + } + + state, err = waitUntilClusterIsProvisioned(vcdClient, d, rde.DefinedEntity.ID) + if err != nil { + return diag.Errorf("Kubernetes cluster update failed: %s", err) + } + if state != "provisioned" { + return diag.Errorf("Kubernetes cluster update failed, cluster is not in 'provisioned' state, but '%s'", state) + }*/ +} + +// marshalMultipleYamlDocuments takes a slice of maps representing multiple YAML documents (one per item in the slice) and +// marshals all of them into a single string with the corresponding separators "---". +func marshalMultipleYamlDocuments(yamlDocuments []map[any]any) (string, error) { + result := "" + for i, yamlDoc := range yamlDocuments { + updatedSingleDoc, err := yaml.Marshal(yamlDoc) + if err != nil { + return "", fmt.Errorf("error marshaling the updated CAPVCD YAML '%v': %s", yamlDoc, err) + } + result += fmt.Sprintf("%s\n", updatedSingleDoc) + if i < len(yamlDocuments)-1 { // The last document doesn't need the YAML separator + result += "---\n" + } + } + return result, nil +} + +// unmarshalMultipleYamlDocuments takes a multi-document YAML (multiple YAML documents are separated by "---") and +// unmarshals all of them into a slice of generic maps with the corresponding content. +func unmarshalMultipleYamlDocuments(yamlDocuments string) ([]map[any]any, error) { + if len(strings.TrimSpace(yamlDocuments)) == 0 { + return []map[any]any{}, nil + } + + dec := yaml.NewDecoder(bytes.NewReader([]byte(yamlDocuments))) + documentCount := strings.Count(yamlDocuments, "---") + if documentCount == 0 { + // If it doesn't have any separator, we can assume it's just a single document. + // Otherwise, it will fail afterward + documentCount = 1 + } + yamlDocs := make([]map[any]any, documentCount) + i := 0 + for i < documentCount { + err := dec.Decode(&yamlDocs[i]) + if err != nil { + return nil, err + } + i++ + } + return yamlDocs, nil +} diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go new file mode 100644 index 000000000..6344abb66 --- /dev/null +++ b/govcd/cse_yaml_unit_test.go @@ -0,0 +1,321 @@ +//go:build unit || ALL + +package govcd + +import ( + "gopkg.in/yaml.v2" + "os" + "reflect" + "strings" + "testing" +) + +func Test_cseUpdateKubernetesTemplateInYaml1(t *testing.T) { + capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") + if err != nil { + t.Fatalf("could not read CAPI YAML test file: %s", err) + } + + yamlDocs, err := unmarshalMultipleYamlDocuments(string(capiYaml)) + if err != nil { + t.Fatalf("could not unmarshal CAPI YAML test file: %s", err) + } + + // We explore the YAML documents to get the OVA template name that will be updated + // with the new one. + oldOvaName := "" + for _, document := range yamlDocs { + if document["kind"] != "VCDMachineTemplate" { + continue + } + + name, err := traverseMapAndGet[string](document, "metadata.name") + if err != nil { + t.Fatalf("expected to find metadata.name in %v but got an error: %s", document, err) + } + + if strings.Contains(name, "control-plane-node-pool") { + oldOvaName, err = traverseMapAndGet[string](document, "spec.template.spec.template") + if err != nil { + t.Fatalf("expected to find spec.template.spec.template in %v but got an error: %s", document, err) + } + } + } + if oldOvaName == "" { + t.Fatalf("the OVA that needs to be changed is empty") + } + + // We call the function to update the old OVA with the new one + newOvaName := "my-super-ova-name" + err = cseUpdateKubernetesTemplateInYaml(yamlDocs, newOvaName) + if err != nil { + t.Fatalf("%s", err) + } + + // We check the status of all the YAML documents. Only the Control Plane template OVA should be changed. + for _, document := range yamlDocs { + if document["kind"] != "VCDMachineTemplate" { + continue + } + + name, err := traverseMapAndGet[string](document, "metadata.name") + if err != nil { + t.Fatalf("expected to find metadata.name in %v but got an error: %s", document, err) + } + b, err := yaml.Marshal(document) + if err != nil { + t.Fatalf("error marshaling %v: %s", document, err) + } + if strings.Contains(name, "control-plane-node-pool") { + // If the document is a Control Plane, the old template must not appear, only the new one can be there + if !strings.Contains(string(b), newOvaName) && strings.Contains(string(b), oldOvaName) { + t.Fatalf("failed updating the Kubernetes OVA template in the Control Plane:\n%s", b) + } + } else { + // If the document is any other pool, the old template should remain untouched + if strings.Contains(string(b), newOvaName) && !strings.Contains(string(b), oldOvaName) { + t.Fatalf("it updated the Kubernetes OVA template in a wrong YAML document:\n%s", b) + } + } + } +} + +// Test_unmarshalMultplieYamlDocuments tests the unmarshalling of multiple YAML documents with unmarshalMultplieYamlDocuments +func Test_unmarshalMultplieYamlDocuments(t *testing.T) { + capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") + if err != nil { + t.Fatalf("could not read YAML test file: %s", err) + } + + tests := []struct { + name string + yamlDocuments string + want int + wantErr bool + }{ + { + name: "unmarshal correct amount of documents", + yamlDocuments: string(capiYaml), + want: 9, + wantErr: false, + }, + { + name: "unmarshal single yaml document", + yamlDocuments: "test: foo", + want: 1, + wantErr: false, + }, + { + name: "unmarshal empty yaml document", + yamlDocuments: "", + want: 0, + }, + { + name: "unmarshal wrong yaml document", + yamlDocuments: "thisIsNotAYaml", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := unmarshalMultipleYamlDocuments(tt.yamlDocuments) + if (err != nil) != tt.wantErr { + t.Errorf("unmarshalMultplieYamlDocuments() error = %v, wantErr %v", err, tt.wantErr) + return + } + if len(got) != tt.want { + t.Errorf("unmarshalMultplieYamlDocuments() got %d documents, want %d", len(got), tt.want) + } + }) + } +} + +// Test_marshalMultplieYamlDocuments tests the marshalling of multiple YAML documents with marshalMultplieYamlDocuments +func Test_marshalMultplieYamlDocuments(t *testing.T) { + capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") + if err != nil { + t.Fatalf("could not read YAML test file: %s", err) + } + + unmarshaledCapiYaml, err := unmarshalMultipleYamlDocuments(string(capiYaml)) + if err != nil { + t.Fatalf("could not unmarshal the YAML test file: %s", err) + } + + tests := []struct { + name string + yamlDocuments []map[any]any + want []map[any]any + wantErr bool + }{ + { + name: "marshal correct amount of documents", + yamlDocuments: unmarshaledCapiYaml, + want: unmarshaledCapiYaml, + wantErr: false, + }, + { + name: "marshal empty slice", + yamlDocuments: []map[any]any{}, + want: []map[any]any{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := marshalMultipleYamlDocuments(tt.yamlDocuments) + if (err != nil) != tt.wantErr { + t.Errorf("marshalMultipleYamlDocuments() error = %v, wantErr %v", err, tt.wantErr) + return + } + gotUnmarshaled, err := unmarshalMultipleYamlDocuments(got) // We unmarshal the result to compare it exactly with DeepEqual + if err != nil { + t.Errorf("unmarshalMultipleYamlDocuments() failed %s", err) + return + } + if !reflect.DeepEqual(gotUnmarshaled, tt.want) { + t.Errorf("marshalMultipleYamlDocuments() got =\n%v, want =\n%v", gotUnmarshaled, tt.want) + } + }) + } +} + +// Test_traverseMapAndGet tests traverseMapAndGet function +func Test_traverseMapAndGet(t *testing.T) { + type args struct { + input any + path string + } + tests := []struct { + name string + args args + wantType string + want any + wantErr string + }{ + { + name: "input is nil", + args: args{ + input: nil, + }, + wantErr: "the input is nil", + }, + { + name: "input is not a map", + args: args{ + input: "error", + }, + wantErr: "the input is a string, not a map[any]any", + }, + { + name: "map is empty", + args: args{ + input: map[any]any{}, + }, + wantErr: "the map is empty", + }, + { + name: "map does not have key", + args: args{ + input: map[any]any{ + "keyA": "value", + }, + path: "keyB", + }, + wantErr: "key 'keyB' does not exist in input map", + }, + { + name: "map has a single simple key", + args: args{ + input: map[any]any{ + "keyA": "value", + }, + path: "keyA", + }, + wantType: "string", + want: "value", + }, + { + name: "map has a single complex key", + args: args{ + input: map[any]any{ + "keyA": map[any]any{ + "keyB": "value", + }, + }, + path: "keyA", + }, + wantType: "map", + want: map[any]any{ + "keyB": "value", + }, + }, + { + name: "map has a complex structure", + args: args{ + input: map[any]any{ + "keyA": map[any]any{ + "keyB": map[any]any{ + "keyC": "value", + }, + }, + }, + path: "keyA.keyB.keyC", + }, + wantType: "string", + want: "value", + }, + { + name: "requested path is deeper than the map structure", + args: args{ + input: map[any]any{ + "keyA": map[any]any{ + "keyB": map[any]any{ + "keyC": "value", + }, + }, + }, + path: "keyA.keyB.keyC.keyD", + }, + wantErr: "key 'keyC' is a string, not a map, but there are still 1 paths to explore", + }, + { + name: "obtained value does not correspond to the desired type", + args: args{ + input: map[any]any{ + "keyA": map[any]any{ + "keyB": map[any]any{ + "keyC": map[any]any{}, + }, + }, + }, + path: "keyA.keyB.keyC", + }, + wantType: "string", + wantErr: "could not convert obtained type map[string]interface {} to requested string", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got any + var err error + if tt.wantType == "string" { + got, err = traverseMapAndGet[string](tt.args.input, tt.args.path) + } else if tt.wantType == "map" { + got, err = traverseMapAndGet[map[any]any](tt.args.input, tt.args.path) + } else { + t.Fatalf("wantType type not used in this test") + } + + if err != nil { + if tt.wantErr != err.Error() { + t.Errorf("traverseMapAndGet() error = %v, wantErr = %v", err, tt.wantErr) + } + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("traverseMapAndGet() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/govcd/test-resources/capiYaml.yaml b/govcd/test-resources/capiYaml.yaml new file mode 100644 index 000000000..1e4a7aacc --- /dev/null +++ b/govcd/test-resources/capiYaml.yaml @@ -0,0 +1,210 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "test1" + namespace: "test1-ns" + labels: + clusterctl.cluster.x-k8s.io: "" + clusterctl.cluster.x-k8s.io/move: "" +spec: + clusterName: "test1" + maxUnhealthy: "100%" + nodeStartupTimeout: "900s" + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: "test1" + unhealthyConditions: + - type: Ready + status: Unknown + timeout: "300s" + - type: Ready + status: "False" + timeout: "300s" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: VCDMachineTemplate +metadata: + name: "node-pool-1" + namespace: "test1-ns" +spec: + template: + spec: + catalog: "tkgm_catalog" + template: "ubuntu-2004-kube-v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc" + sizingPolicy: "TKG small" + placementPolicy: "" + storageProfile: "*" + diskSize: "20Gi" + enableNvidiaGPU: false +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "node-pool-1" + namespace: "test1-ns" +spec: + clusterName: "test1" + replicas: 1 + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: "test1-kct" + namespace: "test1-ns" + clusterName: "test1" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: VCDMachineTemplate + name: "node-pool-1" + namespace: "test1-ns" + version: "v1.25.7+vmware.2" +--- + +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "test1" + namespace: "test1-ns" + labels: + cluster-role.tkg.tanzu.vmware.com/management: "" + tanzuKubernetesRelease: "v1.25.7---vmware.2-tkg.1" + tkg.tanzu.vmware.com/cluster-name: "test1" + annotations: + osInfo: "ubuntu,20.04,amd64" + TKGVERSION: "v2.2.0" +spec: + clusterNetwork: + pods: + cidrBlocks: + - "100.96.0.0/11" + serviceDomain: cluster.local + services: + cidrBlocks: + - "100.64.0.0/13" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "test1-control-plane-node-pool" + namespace: "test1-ns" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: VCDCluster + name: "test1" + namespace: "test1-ns" +--- +apiVersion: v1 +kind: Secret +metadata: + name: capi-user-credentials + namespace: test1-ns +type: Opaque +data: + username: "ZHVtbXkK" + refreshToken: "ZHVtbXkK" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: VCDCluster +metadata: + name: "test1" + namespace: "test1-ns" +spec: + site: "https://www.my-vcd-instance.com" + org: "tenant_org" + ovdc: "tenant_vdc" + ovdcNetwork: "tenant_net_routed" + useAsManagementCluster: false + userContext: + secretRef: + name: capi-user-credentials + namespace: "test1-ns" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: VCDMachineTemplate +metadata: + name: "test1-control-plane-node-pool" + namespace: "test1-ns" +spec: + template: + spec: + catalog: "tkgm_catalog" + template: "ubuntu-2004-kube-v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc" + sizingPolicy: "TKG small" + placementPolicy: "" + storageProfile: "*" + diskSize: 20Gi +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "test1-control-plane-node-pool" + namespace: "test1-ns" +spec: + kubeadmConfigSpec: + preKubeadmCommands: + - mv /etc/ssl/certs/custom_certificate_*.crt /usr/local/share/ca-certificates && update-ca-certificates + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + dns: + imageRepository: "projects.registry.vmware.com/tkg" + imageTag: "v1.9.3_vmware.8" + etcd: + local: + imageRepository: "projects.registry.vmware.com/tkg" + imageTag: "v3.5.6_vmware.9" + imageRepository: "projects.registry.vmware.com/tkg" + users: + - name: root + sshAuthorizedKeys: + - "" + initConfiguration: + nodeRegistration: + criSocket: /run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%%,nodefs.inodesFree<0%%,imagefs.available<0%% + cloud-provider: external + joinConfiguration: + nodeRegistration: + criSocket: /run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%%,nodefs.inodesFree<0%%,imagefs.available<0%% + cloud-provider: external + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: VCDMachineTemplate + name: "test1-control-plane-node-pool" + namespace: "test1-ns" + replicas: 1 + version: "v1.25.7+vmware.2" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "test1-kct" + namespace: "test1-ns" +spec: + template: + spec: + users: + - name: root + sshAuthorizedKeys: + - "" + useExperimentalRetryJoin: true + preKubeadmCommands: + - mv /etc/ssl/certs/custom_certificate_*.crt /usr/local/share/ca-certificates && update-ca-certificates + joinConfiguration: + nodeRegistration: + criSocket: /run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%%,nodefs.inodesFree<0%%,imagefs.available<0%% + cloud-provider: external From 93ff0c61d226adb8440b4bbe0123645158dde1f9 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 31 Jan 2024 13:12:14 +0100 Subject: [PATCH 009/115] Fix bug Signed-off-by: abarreiro --- govcd/cse.go | 2 +- govcd/cse_yaml.go | 28 ++++++++++------------ govcd/cse_yaml_unit_test.go | 47 ++++++++++--------------------------- 3 files changed, 25 insertions(+), 52 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 66aa5ca30..9ff533215 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -253,7 +253,7 @@ func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput, if input.AutoRepairOnErrors != nil { cluster.Capvcd.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors } - updatedCapiYaml, err := cseUpdateCapiYaml(cluster.Capvcd.Spec.CapiYaml, input) + updatedCapiYaml, err := cseUpdateCapiYaml(cluster.client, cluster.Capvcd.Spec.CapiYaml, input) if err != nil { return err } diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 145c0967f..0f9ba3a2e 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -53,23 +53,15 @@ func traverseMapAndGet[ResultType any](input interface{}, path string) (ResultTy return resultTyped, nil } +// cseUpdateKubernetesTemplateInYaml updates the Kubernetes template OVA used by all the VCDMachineTemplate blocks func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[any]any, kubernetesTemplateOvaName string) error { updated := false for _, d := range yamlDocuments { if d["kind"] != "VCDMachineTemplate" { continue } - // Check that it is a control plane pool by checking the name - name, err := traverseMapAndGet[string](d, "metadata.name") - if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) - } - if !strings.Contains(name, "control-plane-node-pool") { - continue - } - // Perform the update - _, err = traverseMapAndGet[string](d, "spec.template.spec.template") + _, err := traverseMapAndGet[string](d, "spec.template.spec.template") if err != nil { return fmt.Errorf("incorrect CAPI YAML: %s", err) } @@ -77,7 +69,7 @@ func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[any]any, kubernetesTe updated = true } if !updated { - return fmt.Errorf("could not find any Control Plane pool in the CAPI YAML") + return fmt.Errorf("could not find any template inside the VCDMachineTemplate blocks in the CAPI YAML") } return nil } @@ -98,7 +90,7 @@ func updateNodeHealthCheckYaml(docs []map[any]any, b bool) error { // and its Node Health Check capabilities, by using the new values provided as input. // If some of the values of the input is not provided, it doesn't change them. // If none of the values is provided, it just returns the same untouched YAML. -func cseUpdateCapiYaml(capiYaml string, input CseClusterUpdateInput) (string, error) { +func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateInput) (string, error) { if input.ControlPlane == nil && input.WorkerPools == nil && input.NodeHealthCheck == nil && input.KubernetesTemplateOvaId == nil { return capiYaml, nil } @@ -107,17 +99,21 @@ func cseUpdateCapiYaml(capiYaml string, input CseClusterUpdateInput) (string, er // document it finds. yamlDocs, err := unmarshalMultipleYamlDocuments(capiYaml) if err != nil { - return "", fmt.Errorf("error unmarshaling CAPI YAML: %s", err) + return capiYaml, fmt.Errorf("error unmarshaling CAPI YAML: %s", err) } - // As a side note, we can't optimize this one with a 'if currentValue equals updatedValue do nothing' because + // As a side note, we can't optimize this one with "if equals do nothing" because // in order to retrieve the current value we would need to explore the YAML anyway, which is what we also need to do to update it. // So in this special case this "optimization" would optimize nothing. The same happens with other YAML values. if input.KubernetesTemplateOvaId != nil { + vAppTemplate, err := getVAppTemplateById(client, *input.KubernetesTemplateOvaId) + if err != nil { + return capiYaml, fmt.Errorf("could not retrieve the Kubernetes OVA with ID '%s': %s", *input.KubernetesTemplateOvaId, err) + } - err := cseUpdateKubernetesTemplateInYaml(yamlDocs, *input.KubernetesTemplateOvaId) + err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate.Name) if err != nil { - return "", err + return capiYaml, err } } diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 6344abb66..66f9fedaa 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -3,14 +3,14 @@ package govcd import ( - "gopkg.in/yaml.v2" "os" "reflect" "strings" "testing" ) -func Test_cseUpdateKubernetesTemplateInYaml1(t *testing.T) { +// Test_cseUpdateKubernetesTemplateInYaml tests the update process of the Kubernetes template OVA in a CAPI YAML. +func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") if err != nil { t.Fatalf("could not read CAPI YAML test file: %s", err) @@ -29,17 +29,11 @@ func Test_cseUpdateKubernetesTemplateInYaml1(t *testing.T) { continue } - name, err := traverseMapAndGet[string](document, "metadata.name") + oldOvaName, err = traverseMapAndGet[string](document, "spec.template.spec.template") if err != nil { - t.Fatalf("expected to find metadata.name in %v but got an error: %s", document, err) - } - - if strings.Contains(name, "control-plane-node-pool") { - oldOvaName, err = traverseMapAndGet[string](document, "spec.template.spec.template") - if err != nil { - t.Fatalf("expected to find spec.template.spec.template in %v but got an error: %s", document, err) - } + t.Fatalf("expected to find spec.template.spec.template in %v but got an error: %s", document, err) } + break } if oldOvaName == "" { t.Fatalf("the OVA that needs to be changed is empty") @@ -52,31 +46,14 @@ func Test_cseUpdateKubernetesTemplateInYaml1(t *testing.T) { t.Fatalf("%s", err) } - // We check the status of all the YAML documents. Only the Control Plane template OVA should be changed. - for _, document := range yamlDocs { - if document["kind"] != "VCDMachineTemplate" { - continue - } + updatedYaml, err := marshalMultipleYamlDocuments(yamlDocs) + if err != nil { + t.Fatalf("error marshaling %v: %s", yamlDocs, err) + } - name, err := traverseMapAndGet[string](document, "metadata.name") - if err != nil { - t.Fatalf("expected to find metadata.name in %v but got an error: %s", document, err) - } - b, err := yaml.Marshal(document) - if err != nil { - t.Fatalf("error marshaling %v: %s", document, err) - } - if strings.Contains(name, "control-plane-node-pool") { - // If the document is a Control Plane, the old template must not appear, only the new one can be there - if !strings.Contains(string(b), newOvaName) && strings.Contains(string(b), oldOvaName) { - t.Fatalf("failed updating the Kubernetes OVA template in the Control Plane:\n%s", b) - } - } else { - // If the document is any other pool, the old template should remain untouched - if strings.Contains(string(b), newOvaName) && !strings.Contains(string(b), oldOvaName) { - t.Fatalf("it updated the Kubernetes OVA template in a wrong YAML document:\n%s", b) - } - } + // No document should have the old OVA + if !strings.Contains(updatedYaml, newOvaName) || strings.Contains(updatedYaml, oldOvaName) { + t.Fatalf("failed updating the Kubernetes OVA template in the Control Plane:\n%s", updatedYaml) } } From 65a145eb9b3f340e4cbda642bfc5bd2eb97d9def Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 31 Jan 2024 13:45:21 +0100 Subject: [PATCH 010/115] Add more unit tests for worker pool update Signed-off-by: abarreiro --- govcd/cse.go | 1 + govcd/cse_yaml.go | 54 +++++++++++++++++++++-- govcd/cse_yaml_unit_test.go | 87 +++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 9ff533215..946753f78 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -92,6 +92,7 @@ type CseClusterUpdateInput struct { KubernetesTemplateOvaId *string ControlPlane *CseControlPlaneUpdateInput WorkerPools *map[string]CseWorkerPoolUpdateInput // Maps a node pool name with its contents + NewWorkerPools *[]CseWorkerPoolCreateInput NodeHealthCheck *bool AutoRepairOnErrors *bool } diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 0f9ba3a2e..ecc99ddb0 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -78,7 +78,46 @@ func updateControlPlaneYaml(docs []map[any]any, input CseControlPlaneUpdateInput return nil } -func updateWorkerPoolsYaml(docs []map[any]any, m map[string]CseWorkerPoolUpdateInput) error { +func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[any]any, workerPools map[string]CseWorkerPoolUpdateInput) error { + updated := 0 + for _, d := range yamlDocuments { + if d["kind"] != "MachineDeployment" { + continue + } + + workerPoolName, err := traverseMapAndGet[string](d, "metadata.name") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + + workerPoolToUpdate := "" + for wpName := range workerPools { + if wpName == workerPoolName { + workerPoolToUpdate = wpName + } + } + // This worker pool is not going to be updated, continue searching for another one + if workerPoolToUpdate == "" { + continue + } + + _, err = traverseMapAndGet[int](d, "spec.replicas") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + if workerPools[workerPoolToUpdate].MachineCount < 0 { + return fmt.Errorf("incorrect machine count for worker pool %s: %d. Should be at least 0", workerPoolToUpdate, workerPools[workerPoolToUpdate].MachineCount) + } + d["spec"].(map[any]any)["replicas"] = workerPools[workerPoolToUpdate].MachineCount + updated++ + } + if updated != len(workerPools) { + return fmt.Errorf("could not update all the Node pools. Updated %d, expected %d", updated, len(workerPools)) + } + return nil +} + +func addWorkerPoolsYaml(docs []map[any]any, inputs []CseWorkerPoolCreateInput) error { return nil } @@ -120,14 +159,21 @@ func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateIn if input.ControlPlane != nil { err := updateControlPlaneYaml(yamlDocs, *input.ControlPlane) if err != nil { - return "", err + return capiYaml, err } } if input.WorkerPools != nil { - err := updateWorkerPoolsYaml(yamlDocs, *input.WorkerPools) + err := cseUpdateWorkerPoolsInYaml(yamlDocs, *input.WorkerPools) if err != nil { - return "", err + return capiYaml, err + } + } + + if input.NewWorkerPools != nil { + err := addWorkerPoolsYaml(yamlDocs, *input.NewWorkerPools) + if err != nil { + return capiYaml, err } } diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 66f9fedaa..f6ffeb520 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -57,6 +57,93 @@ func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { } } +// Test_cseUpdateWorkerPoolsInYaml tests the update process of the Worker pools in a CAPI YAML. +func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { + capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") + if err != nil { + t.Fatalf("could not read CAPI YAML test file: %s", err) + } + + yamlDocs, err := unmarshalMultipleYamlDocuments(string(capiYaml)) + if err != nil { + t.Fatalf("could not unmarshal CAPI YAML test file: %s", err) + } + // We explore the YAML documents to get the OVA template name that will be updated + // with the new one. + oldNodePools := map[string]CseWorkerPoolUpdateInput{} + for _, document := range yamlDocs { + if document["kind"] != "MachineDeployment" { + continue + } + + workerPoolName, err := traverseMapAndGet[string](document, "metadata.name") + if err != nil { + t.Fatalf("incorrect CAPI YAML: %s", err) + } + + oldReplicas, err := traverseMapAndGet[int](document, "spec.replicas") + if err != nil { + t.Fatalf("incorrect CAPI YAML: %s", err) + } + oldNodePools[workerPoolName] = CseWorkerPoolUpdateInput{ + MachineCount: oldReplicas, + } + } + if len(oldNodePools) == -1 { + t.Fatalf("didn't get any valid worker node pool") + } + + // We call the function to update the old pools with the new ones + newReplicas := 66 + newNodePools := map[string]CseWorkerPoolUpdateInput{} + for name := range oldNodePools { + newNodePools[name] = CseWorkerPoolUpdateInput{ + MachineCount: newReplicas, + } + } + err = cseUpdateWorkerPoolsInYaml(yamlDocs, newNodePools) + if err != nil { + t.Fatalf("%s", err) + } + + // The worker pools should have now the new details updated + for _, document := range yamlDocs { + if document["kind"] != "MachineDeployment" { + continue + } + + retrievedReplicas, err := traverseMapAndGet[int](document, "spec.replicas") + if err != nil { + t.Fatalf("incorrect CAPI YAML: %s", err) + } + if retrievedReplicas != newReplicas { + t.Fatalf("expected %d replicas but got %d", newReplicas, retrievedReplicas) + } + } + + // Corner case: Wrong replicas + newReplicas = -1 + newNodePools = map[string]CseWorkerPoolUpdateInput{} + for name := range oldNodePools { + newNodePools[name] = CseWorkerPoolUpdateInput{ + MachineCount: newReplicas, + } + } + err = cseUpdateWorkerPoolsInYaml(yamlDocs, newNodePools) + if err == nil { + t.Fatal("Expected an error, but got none") + } + + // Corner case: No worker pool with that name exists + newNodePools = map[string]CseWorkerPoolUpdateInput{ + "not-exist": {}, + } + err = cseUpdateWorkerPoolsInYaml(yamlDocs, newNodePools) + if err == nil { + t.Fatal("Expected an error, but got none") + } +} + // Test_unmarshalMultplieYamlDocuments tests the unmarshalling of multiple YAML documents with unmarshalMultplieYamlDocuments func Test_unmarshalMultplieYamlDocuments(t *testing.T) { capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") From c4774cd1e1a8669a37c19ef03efe3723260c7705 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 31 Jan 2024 16:52:53 +0100 Subject: [PATCH 011/115] Continue with update, but Reads need a revisit Signed-off-by: abarreiro --- govcd/cse.go | 64 +++++++++++++++-------------------- govcd/cse_test.go | 6 ++-- govcd/cse_yaml.go | 32 +++++++++++++++--- govcd/cse_yaml_unit_test.go | 67 +++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 45 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 946753f78..0051c51f3 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -30,6 +30,9 @@ type CseClusterApiProviderCluster struct { Owner string Etag string client *Client + // TODO: Updated fields are inside the YAML file, like if you update the Control Plane replicas, you need to inspect + // the YAML to get the updated value. Inspecting Capvcd fields will do nothing. So I need to put here this information + // for convenience. } // CseClusterCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods @@ -187,59 +190,54 @@ func (cluster *CseClusterApiProviderCluster) Refresh() error { return nil } -// UpdateWorkerPools executes a synchronous update on the receiver cluster to change the worker pools. If the given -// timeout is 0, it waits forever for the cluster update to finish. Otherwise, if the timeout is reached and the cluster is not available, -// it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster will be available in the -// receiver CseClusterApiProviderCluster. -func (cluster *CseClusterApiProviderCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput, timeoutMinutes time.Duration) error { +// UpdateWorkerPools executes an update on the receiver cluster to change the existing worker pools. +func (cluster *CseClusterApiProviderCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput) error { return cluster.Update(CseClusterUpdateInput{ WorkerPools: &input, - }, timeoutMinutes) + }) } -// UpdateControlPlane executes a synchronous update on the receiver cluster to change the control plane. If the given -// timeout is 0, it waits forever for the cluster update to finish. Otherwise, if the timeout is reached and the cluster is not available, -// it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster will be available in the -// receiver CseClusterApiProviderCluster. -func (cluster *CseClusterApiProviderCluster) UpdateControlPlane(input CseControlPlaneUpdateInput, timeoutMinutes time.Duration) error { +// AddWorkerPools executes an update on the receiver cluster to add new worker pools. +func (cluster *CseClusterApiProviderCluster) AddWorkerPools(input []CseWorkerPoolCreateInput) error { + return cluster.Update(CseClusterUpdateInput{ + NewWorkerPools: &input, + }) +} + +// UpdateControlPlane executes an update on the receiver cluster to change the existing control plane. +func (cluster *CseClusterApiProviderCluster) UpdateControlPlane(input CseControlPlaneUpdateInput) error { return cluster.Update(CseClusterUpdateInput{ ControlPlane: &input, - }, timeoutMinutes) + }) } -// ChangeKubernetesTemplate executes a synchronous update on the receiver cluster to change the Kubernetes template of the cluster. If the given -// timeout is 0, it waits forever for the cluster update to finish. Otherwise, if the timeout is reached and the cluster is not available, -// it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster will be available in the -// receiver CseClusterApiProviderCluster. -func (cluster *CseClusterApiProviderCluster) ChangeKubernetesTemplate(kubernetesTemplateOvaId string, timeoutMinutes time.Duration) error { +// ChangeKubernetesTemplate executes an update on the receiver cluster to change the Kubernetes template of the cluster. +func (cluster *CseClusterApiProviderCluster) ChangeKubernetesTemplate(kubernetesTemplateOvaId string) error { return cluster.Update(CseClusterUpdateInput{ KubernetesTemplateOvaId: &kubernetesTemplateOvaId, - }, timeoutMinutes) + }) } -// SetHealthCheck executes a synchronous update on the receiver cluster to enable or disable the machine health check capabilities. If the given -// timeout is 0, it waits forever for the cluster update to finish. Otherwise, if the timeout is reached and the cluster is not available, -// it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster will be available in the -// receiver CseClusterApiProviderCluster. -func (cluster *CseClusterApiProviderCluster) SetHealthCheck(healthCheckEnabled bool, timeoutMinutes time.Duration) error { +// SetHealthCheck executes an update on the receiver cluster to enable or disable the machine health check capabilities. +func (cluster *CseClusterApiProviderCluster) SetHealthCheck(healthCheckEnabled bool) error { return cluster.Update(CseClusterUpdateInput{ NodeHealthCheck: &healthCheckEnabled, - }, timeoutMinutes) + }) } -// SetAutoRepairOnErrors executes a synchronous update on the receiver cluster to change the flag that controls the auto-repair +// SetAutoRepairOnErrors executes an update on the receiver cluster to change the flag that controls the auto-repair // capabilities of CSE. func (cluster *CseClusterApiProviderCluster) SetAutoRepairOnErrors(autoRepairOnErrors bool) error { return cluster.Update(CseClusterUpdateInput{ AutoRepairOnErrors: &autoRepairOnErrors, - }, 0) + }) } // Update executes a synchronous update on the receiver cluster to perform a update on any of the allowed parameters of the cluster. If the given // timeout is 0, it waits forever for the cluster update to finish. Otherwise, if the timeout is reached and the cluster is not available, // it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster will be available in the // receiver CseClusterApiProviderCluster. -func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput, timeoutMinutes time.Duration) error { +func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput) error { err := cluster.Refresh() if err != nil { return err @@ -307,18 +305,10 @@ func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput, } if !updated { - return fmt.Errorf("could not update the Kubernetes cluster '%s' due to %d ETag locks that blocked the operation", cluster.ID, maxRetries) + return fmt.Errorf("could not update the Kubernetes cluster '%s' after %d retries, due to an ETag lock blocking the operations", cluster.ID, maxRetries) } - _, finalError := waitUntilClusterIsProvisioned(cluster.client, cluster.ID, timeoutMinutes) - - // We do a Refresh() even if the cluster update ended with errors, so the receiver entity gets updated with latest fields. - // Then, if the update ended with errors, we return them - _ = cluster.Refresh() - if finalError != nil { - return finalError - } - return nil + return cluster.Refresh() } // Delete deletes a CSE Kubernetes cluster, waiting the specified amount of minutes. If the timeout is reached, this method diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 51293360e..f6b0cd509 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -124,7 +124,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { } } // Perform the update - err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, 0) + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}) check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up @@ -152,10 +152,10 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { cluster, err := org.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:e8e82bcc-50a1-484f-9dd0-20965ab3e865") check.Assert(err, IsNil) - workerPoolName := "worker-node-pool-1" + workerPoolName := "cse-test1-worker-node-pool-1" // Perform the update - err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, 0) + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}) check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index ecc99ddb0..efa1df8f2 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -74,7 +74,27 @@ func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[any]any, kubernetesTe return nil } -func updateControlPlaneYaml(docs []map[any]any, input CseControlPlaneUpdateInput) error { +func cseUpdateControlPlaneInYaml(yamlDocuments []map[any]any, input CseControlPlaneUpdateInput) error { + if input.MachineCount < 0 { + return fmt.Errorf("incorrect machine count for Control Plane: %d. Should be at least 0", input.MachineCount) + } + + updated := false + for _, d := range yamlDocuments { + if d["kind"] != "KubeadmControlPlane" { + continue + } + + _, err := traverseMapAndGet[int](d, "spec.replicas") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + d["spec"].(map[any]any)["replicas"] = input.MachineCount + updated = true + } + if !updated { + return fmt.Errorf("could not update the KubeadmControlPlane block in the CAPI YAML") + } return nil } @@ -101,13 +121,15 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[any]any, workerPools map[str continue } + if workerPools[workerPoolToUpdate].MachineCount < 0 { + return fmt.Errorf("incorrect machine count for worker pool %s: %d. Should be at least 0", workerPoolToUpdate, workerPools[workerPoolToUpdate].MachineCount) + } + _, err = traverseMapAndGet[int](d, "spec.replicas") if err != nil { return fmt.Errorf("incorrect CAPI YAML: %s", err) } - if workerPools[workerPoolToUpdate].MachineCount < 0 { - return fmt.Errorf("incorrect machine count for worker pool %s: %d. Should be at least 0", workerPoolToUpdate, workerPools[workerPoolToUpdate].MachineCount) - } + d["spec"].(map[any]any)["replicas"] = workerPools[workerPoolToUpdate].MachineCount updated++ } @@ -157,7 +179,7 @@ func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateIn } if input.ControlPlane != nil { - err := updateControlPlaneYaml(yamlDocs, *input.ControlPlane) + err := cseUpdateControlPlaneInYaml(yamlDocs, *input.ControlPlane) if err != nil { return capiYaml, err } diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index f6ffeb520..5837d1adb 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -144,6 +144,73 @@ func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { } } +// Test_cseUpdateControlPlaneInYaml tests the update process of the Control Plane in a CAPI YAML. +func Test_cseUpdateControlPlaneInYaml(t *testing.T) { + capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") + if err != nil { + t.Fatalf("could not read CAPI YAML test file: %s", err) + } + + yamlDocs, err := unmarshalMultipleYamlDocuments(string(capiYaml)) + if err != nil { + t.Fatalf("could not unmarshal CAPI YAML test file: %s", err) + } + // We explore the YAML documents to get the OVA template name that will be updated + // with the new one. + oldControlPlane := CseControlPlaneUpdateInput{} + for _, document := range yamlDocs { + if document["kind"] != "KubeadmControlPlane" { + continue + } + + oldReplicas, err := traverseMapAndGet[int](document, "spec.replicas") + if err != nil { + t.Fatalf("incorrect CAPI YAML: %s", err) + } + oldControlPlane = CseControlPlaneUpdateInput{ + MachineCount: oldReplicas, + } + } + if reflect.DeepEqual(oldControlPlane, CseWorkerPoolUpdateInput{}) { + t.Fatalf("didn't get any valid Control Plane") + } + + // We call the function to update the old pools with the new ones + newReplicas := 66 + newControlPlane := CseControlPlaneUpdateInput{ + MachineCount: newReplicas, + } + err = cseUpdateControlPlaneInYaml(yamlDocs, newControlPlane) + if err != nil { + t.Fatalf("%s", err) + } + + // The worker pools should have now the new details updated + for _, document := range yamlDocs { + if document["kind"] != "KubeadmControlPlane" { + continue + } + + retrievedReplicas, err := traverseMapAndGet[int](document, "spec.replicas") + if err != nil { + t.Fatalf("incorrect CAPI YAML: %s", err) + } + if retrievedReplicas != newReplicas { + t.Fatalf("expected %d replicas but got %d", newReplicas, retrievedReplicas) + } + } + + // Corner case: Wrong replicas + newReplicas = -1 + newControlPlane = CseControlPlaneUpdateInput{ + MachineCount: newReplicas, + } + err = cseUpdateControlPlaneInYaml(yamlDocs, newControlPlane) + if err == nil { + t.Fatal("Expected an error, but got none") + } +} + // Test_unmarshalMultplieYamlDocuments tests the unmarshalling of multiple YAML documents with unmarshalMultplieYamlDocuments func Test_unmarshalMultplieYamlDocuments(t *testing.T) { capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") From c4a5c28d04e60c602ee8cbb35f9fe898e206cd61 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 1 Feb 2024 12:02:35 +0100 Subject: [PATCH 012/115] Change YAML library Signed-off-by: abarreiro --- go.mod | 3 +- go.sum | 4 +++ govcd/cse.go | 1 + govcd/cse_yaml.go | 54 +++++++++++++------------------ govcd/cse_yaml_unit_test.go | 64 ++++++++++++++++++------------------- 5 files changed, 62 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index bc528fbac..bc92f0265 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,8 @@ require ( github.com/kr/pretty v0.2.1 github.com/peterhellberg/link v1.1.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v2 v2.2.2 + sigs.k8s.io/yaml v1.4.0 ) require ( diff --git a/go.sum b/go.sum index 187972327..4a1abc2ac 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c h1:3LdnoQiW6yLkxRIw github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= github.com/go-yaml/yaml/v2 v2.2.2 h1:uw2m9KuKRscWGAkuyoBGQcZSdibhmuXKSJ3+9Tj3zXc= github.com/go-yaml/yaml/v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -20,3 +22,5 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/govcd/cse.go b/govcd/cse.go index 0051c51f3..2b09afcd1 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -355,6 +355,7 @@ func (cluster *CseClusterApiProviderCluster) Delete(timeoutMinutes time.Duration if err != nil { if strings.Contains(strings.ToLower(err.Error()), "etag") { continue // We ignore any ETag error. This just means a clash with the CSE Server, we just try again + // FIXME: No sleep here } return fmt.Errorf("could not mark the Kubernetes cluster with ID '%s' to be deleted: %s", cluster.ID, err) } diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index efa1df8f2..1029fb14e 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -1,9 +1,8 @@ package govcd import ( - "bytes" "fmt" - "gopkg.in/yaml.v2" + "sigs.k8s.io/yaml" "strings" ) @@ -17,7 +16,7 @@ func traverseMapAndGet[ResultType any](input interface{}, path string) (ResultTy if input == nil { return nothing, fmt.Errorf("the input is nil") } - inputMap, ok := input.(map[any]any) + inputMap, ok := input.(map[string]any) if !ok { return nothing, fmt.Errorf("the input is a %T, not a map[string]interface{}", input) } @@ -35,7 +34,7 @@ func traverseMapAndGet[ResultType any](input interface{}, path string) (ResultTy return nothing, fmt.Errorf("key '%s' does not exist in input map", subPath) } if i < len(pathUnits)-1 { - traversedMap, ok := traversed.(map[any]any) + traversedMap, ok := traversed.(map[string]any) if !ok { return nothing, fmt.Errorf("key '%s' is a %T, not a map[string]interface{}, but there are still %d paths to explore", subPath, traversed, len(pathUnits)-(i+1)) } @@ -54,7 +53,7 @@ func traverseMapAndGet[ResultType any](input interface{}, path string) (ResultTy } // cseUpdateKubernetesTemplateInYaml updates the Kubernetes template OVA used by all the VCDMachineTemplate blocks -func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[any]any, kubernetesTemplateOvaName string) error { +func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]any, kubernetesTemplateOvaName string) error { updated := false for _, d := range yamlDocuments { if d["kind"] != "VCDMachineTemplate" { @@ -65,7 +64,7 @@ func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[any]any, kubernetesTe if err != nil { return fmt.Errorf("incorrect CAPI YAML: %s", err) } - d["spec"].(map[any]any)["template"].(map[any]any)["spec"].(map[any]any)["template"] = kubernetesTemplateOvaName + d["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["template"] = kubernetesTemplateOvaName updated = true } if !updated { @@ -74,7 +73,7 @@ func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[any]any, kubernetesTe return nil } -func cseUpdateControlPlaneInYaml(yamlDocuments []map[any]any, input CseControlPlaneUpdateInput) error { +func cseUpdateControlPlaneInYaml(yamlDocuments []map[string]any, input CseControlPlaneUpdateInput) error { if input.MachineCount < 0 { return fmt.Errorf("incorrect machine count for Control Plane: %d. Should be at least 0", input.MachineCount) } @@ -85,11 +84,11 @@ func cseUpdateControlPlaneInYaml(yamlDocuments []map[any]any, input CseControlPl continue } - _, err := traverseMapAndGet[int](d, "spec.replicas") + _, err := traverseMapAndGet[float64](d, "spec.replicas") if err != nil { return fmt.Errorf("incorrect CAPI YAML: %s", err) } - d["spec"].(map[any]any)["replicas"] = input.MachineCount + d["spec"].(map[string]any)["replicas"] = float64(input.MachineCount) // As it was originally unmarshalled as a float64 updated = true } if !updated { @@ -98,7 +97,7 @@ func cseUpdateControlPlaneInYaml(yamlDocuments []map[any]any, input CseControlPl return nil } -func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[any]any, workerPools map[string]CseWorkerPoolUpdateInput) error { +func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[string]CseWorkerPoolUpdateInput) error { updated := 0 for _, d := range yamlDocuments { if d["kind"] != "MachineDeployment" { @@ -125,12 +124,12 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[any]any, workerPools map[str return fmt.Errorf("incorrect machine count for worker pool %s: %d. Should be at least 0", workerPoolToUpdate, workerPools[workerPoolToUpdate].MachineCount) } - _, err = traverseMapAndGet[int](d, "spec.replicas") + _, err = traverseMapAndGet[float64](d, "spec.replicas") if err != nil { return fmt.Errorf("incorrect CAPI YAML: %s", err) } - d["spec"].(map[any]any)["replicas"] = workerPools[workerPoolToUpdate].MachineCount + d["spec"].(map[string]any)["replicas"] = float64(workerPools[workerPoolToUpdate].MachineCount) // As it was originally unmarshalled as a float64 updated++ } if updated != len(workerPools) { @@ -139,11 +138,11 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[any]any, workerPools map[str return nil } -func addWorkerPoolsYaml(docs []map[any]any, inputs []CseWorkerPoolCreateInput) error { +func addWorkerPoolsYaml(docs []map[string]any, inputs []CseWorkerPoolCreateInput) error { return nil } -func updateNodeHealthCheckYaml(docs []map[any]any, b bool) error { +func updateNodeHealthCheckYaml(docs []map[string]any, b bool) error { return nil } @@ -310,7 +309,7 @@ func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateIn // marshalMultipleYamlDocuments takes a slice of maps representing multiple YAML documents (one per item in the slice) and // marshals all of them into a single string with the corresponding separators "---". -func marshalMultipleYamlDocuments(yamlDocuments []map[any]any) (string, error) { +func marshalMultipleYamlDocuments(yamlDocuments []map[string]any) (string, error) { result := "" for i, yamlDoc := range yamlDocuments { updatedSingleDoc, err := yaml.Marshal(yamlDoc) @@ -327,26 +326,19 @@ func marshalMultipleYamlDocuments(yamlDocuments []map[any]any) (string, error) { // unmarshalMultipleYamlDocuments takes a multi-document YAML (multiple YAML documents are separated by "---") and // unmarshals all of them into a slice of generic maps with the corresponding content. -func unmarshalMultipleYamlDocuments(yamlDocuments string) ([]map[any]any, error) { +func unmarshalMultipleYamlDocuments(yamlDocuments string) ([]map[string]any, error) { if len(strings.TrimSpace(yamlDocuments)) == 0 { - return []map[any]any{}, nil + return []map[string]any{}, nil } - dec := yaml.NewDecoder(bytes.NewReader([]byte(yamlDocuments))) - documentCount := strings.Count(yamlDocuments, "---") - if documentCount == 0 { - // If it doesn't have any separator, we can assume it's just a single document. - // Otherwise, it will fail afterward - documentCount = 1 - } - yamlDocs := make([]map[any]any, documentCount) - i := 0 - for i < documentCount { - err := dec.Decode(&yamlDocs[i]) + splitYamlDocs := strings.Split(yamlDocuments, "---\n") + result := make([]map[string]any, len(splitYamlDocs)) + for i, yamlDoc := range splitYamlDocs { + err := yaml.Unmarshal([]byte(yamlDoc), &result[i]) if err != nil { - return nil, err + return nil, fmt.Errorf("could not unmarshal document %s: %s", yamlDoc, err) } - i++ } - return yamlDocs, nil + + return result, nil } diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 5837d1adb..7b534072e 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -81,12 +81,12 @@ func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { t.Fatalf("incorrect CAPI YAML: %s", err) } - oldReplicas, err := traverseMapAndGet[int](document, "spec.replicas") + oldReplicas, err := traverseMapAndGet[float64](document, "spec.replicas") if err != nil { t.Fatalf("incorrect CAPI YAML: %s", err) } oldNodePools[workerPoolName] = CseWorkerPoolUpdateInput{ - MachineCount: oldReplicas, + MachineCount: int(oldReplicas), } } if len(oldNodePools) == -1 { @@ -112,12 +112,12 @@ func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { continue } - retrievedReplicas, err := traverseMapAndGet[int](document, "spec.replicas") + retrievedReplicas, err := traverseMapAndGet[float64](document, "spec.replicas") if err != nil { t.Fatalf("incorrect CAPI YAML: %s", err) } - if retrievedReplicas != newReplicas { - t.Fatalf("expected %d replicas but got %d", newReplicas, retrievedReplicas) + if retrievedReplicas != float64(newReplicas) { + t.Fatalf("expected %d replicas but got %0.f", newReplicas, retrievedReplicas) } } @@ -163,12 +163,12 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { continue } - oldReplicas, err := traverseMapAndGet[int](document, "spec.replicas") + oldReplicas, err := traverseMapAndGet[float64](document, "spec.replicas") if err != nil { t.Fatalf("incorrect CAPI YAML: %s", err) } oldControlPlane = CseControlPlaneUpdateInput{ - MachineCount: oldReplicas, + MachineCount: int(oldReplicas), } } if reflect.DeepEqual(oldControlPlane, CseWorkerPoolUpdateInput{}) { @@ -191,12 +191,12 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { continue } - retrievedReplicas, err := traverseMapAndGet[int](document, "spec.replicas") + retrievedReplicas, err := traverseMapAndGet[float64](document, "spec.replicas") if err != nil { t.Fatalf("incorrect CAPI YAML: %s", err) } - if retrievedReplicas != newReplicas { - t.Fatalf("expected %d replicas but got %d", newReplicas, retrievedReplicas) + if retrievedReplicas != float64(newReplicas) { + t.Fatalf("expected %d replicas but got %0.f", newReplicas, retrievedReplicas) } } @@ -275,8 +275,8 @@ func Test_marshalMultplieYamlDocuments(t *testing.T) { tests := []struct { name string - yamlDocuments []map[any]any - want []map[any]any + yamlDocuments []map[string]any + want []map[string]any wantErr bool }{ { @@ -287,8 +287,8 @@ func Test_marshalMultplieYamlDocuments(t *testing.T) { }, { name: "marshal empty slice", - yamlDocuments: []map[any]any{}, - want: []map[any]any{}, + yamlDocuments: []map[string]any{}, + want: []map[string]any{}, wantErr: false, }, } @@ -336,19 +336,19 @@ func Test_traverseMapAndGet(t *testing.T) { args: args{ input: "error", }, - wantErr: "the input is a string, not a map[any]any", + wantErr: "the input is a string, not a map[string]any", }, { name: "map is empty", args: args{ - input: map[any]any{}, + input: map[string]any{}, }, wantErr: "the map is empty", }, { name: "map does not have key", args: args{ - input: map[any]any{ + input: map[string]any{ "keyA": "value", }, path: "keyB", @@ -358,7 +358,7 @@ func Test_traverseMapAndGet(t *testing.T) { { name: "map has a single simple key", args: args{ - input: map[any]any{ + input: map[string]any{ "keyA": "value", }, path: "keyA", @@ -369,24 +369,24 @@ func Test_traverseMapAndGet(t *testing.T) { { name: "map has a single complex key", args: args{ - input: map[any]any{ - "keyA": map[any]any{ + input: map[string]any{ + "keyA": map[string]any{ "keyB": "value", }, }, path: "keyA", }, wantType: "map", - want: map[any]any{ + want: map[string]any{ "keyB": "value", }, }, { name: "map has a complex structure", args: args{ - input: map[any]any{ - "keyA": map[any]any{ - "keyB": map[any]any{ + input: map[string]any{ + "keyA": map[string]any{ + "keyB": map[string]any{ "keyC": "value", }, }, @@ -399,9 +399,9 @@ func Test_traverseMapAndGet(t *testing.T) { { name: "requested path is deeper than the map structure", args: args{ - input: map[any]any{ - "keyA": map[any]any{ - "keyB": map[any]any{ + input: map[string]any{ + "keyA": map[string]any{ + "keyB": map[string]any{ "keyC": "value", }, }, @@ -413,10 +413,10 @@ func Test_traverseMapAndGet(t *testing.T) { { name: "obtained value does not correspond to the desired type", args: args{ - input: map[any]any{ - "keyA": map[any]any{ - "keyB": map[any]any{ - "keyC": map[any]any{}, + input: map[string]any{ + "keyA": map[string]any{ + "keyB": map[string]any{ + "keyC": map[string]any{}, }, }, }, @@ -433,7 +433,7 @@ func Test_traverseMapAndGet(t *testing.T) { if tt.wantType == "string" { got, err = traverseMapAndGet[string](tt.args.input, tt.args.path) } else if tt.wantType == "map" { - got, err = traverseMapAndGet[map[any]any](tt.args.input, tt.args.path) + got, err = traverseMapAndGet[map[string]any](tt.args.input, tt.args.path) } else { t.Fatalf("wantType type not used in this test") } From 2279a20b1c16f579bf3f8051fe5eb97eb3f7e160 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 1 Feb 2024 13:27:33 +0100 Subject: [PATCH 013/115] Add update MHC Signed-off-by: abarreiro --- govcd/cse.go | 11 ++++++ govcd/cse_template.go | 16 ++++---- govcd/cse_util.go | 29 +++++++------- govcd/cse_yaml.go | 50 ++++++++++++++++++++++-- govcd/cse_yaml_unit_test.go | 77 +++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 25 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 2b09afcd1..992941464 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -30,9 +30,11 @@ type CseClusterApiProviderCluster struct { Owner string Etag string client *Client + // TODO: Updated fields are inside the YAML file, like if you update the Control Plane replicas, you need to inspect // the YAML to get the updated value. Inspecting Capvcd fields will do nothing. So I need to put here this information // for convenience. + CseVersion string } // CseClusterCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods @@ -98,6 +100,10 @@ type CseClusterUpdateInput struct { NewWorkerPools *[]CseWorkerPoolCreateInput NodeHealthCheck *bool AutoRepairOnErrors *bool + + // Private fields that are computed, not requested to the consumer of this struct + vcdKeConfigVersion string + clusterName string } // CseControlPlaneUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods @@ -242,6 +248,7 @@ func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput) if err != nil { return err } + if cluster.Capvcd.Status.VcdKe.State == "" { return fmt.Errorf("can't update a Kubernetes cluster that does not have any state") } @@ -252,6 +259,10 @@ func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput) if input.AutoRepairOnErrors != nil { cluster.Capvcd.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors } + + // Computed attributes that are required, such as the VcdKeConfig version + input.clusterName = cluster.Capvcd.Name + input.vcdKeConfigVersion = cluster.Capvcd.Status.VcdKe.VcdKeVersion updatedCapiYaml, err := cseUpdateCapiYaml(cluster.client, cluster.Capvcd.Spec.CapiYaml, input) if err != nil { return err diff --git a/govcd/cse_template.go b/govcd/cse_template.go index 7a12699f5..f9db7670d 100644 --- a/govcd/cse_template.go +++ b/govcd/cse_template.go @@ -105,12 +105,12 @@ func generateNodePoolYaml(clusterDetails *cseClusterCreationGoTemplateArguments) } // generateMemoryHealthCheckYaml generates a YAML block corresponding to the Kubernetes memory health check. -func generateMemoryHealthCheckYaml(clusterDetails *cseClusterCreationGoTemplateArguments, clusterName string) (string, error) { - if clusterDetails.MachineHealthCheck == nil { +func generateMemoryHealthCheckYaml(mhcSettings *machineHealthCheck, cseVersion, clusterName string) (string, error) { + if mhcSettings == nil { return "", nil } - mhcTmpl, err := getCseTemplate(clusterDetails.CseVersion, "capiyaml_mhc") + mhcTmpl, err := getCseTemplate(cseVersion, "capiyaml_mhc") if err != nil { return "", err } @@ -121,10 +121,10 @@ func generateMemoryHealthCheckYaml(clusterDetails *cseClusterCreationGoTemplateA if err := mhcEmptyTmpl.Execute(buf, map[string]string{ "ClusterName": clusterName, "TargetNamespace": clusterName + "-ns", - "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", clusterDetails.MachineHealthCheck.MaxUnhealthyNodesPercentage), // With the 'percentage' suffix - "NodeStartupTimeout": fmt.Sprintf("%ss", clusterDetails.MachineHealthCheck.NodeStartupTimeout), // With the 'second' suffix - "NodeUnknownTimeout": fmt.Sprintf("%ss", clusterDetails.MachineHealthCheck.NodeUnknownTimeout), // With the 'second' suffix - "NodeNotReadyTimeout": fmt.Sprintf("%ss", clusterDetails.MachineHealthCheck.NodeNotReadyTimeout), // With the 'second' suffix + "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", mhcSettings.MaxUnhealthyNodesPercentage), // With the 'percentage' suffix + "NodeStartupTimeout": fmt.Sprintf("%ss", mhcSettings.NodeStartupTimeout), // With the 'second' suffix + "NodeUnknownTimeout": fmt.Sprintf("%ss", mhcSettings.NodeUnknownTimeout), // With the 'second' suffix + "NodeNotReadyTimeout": fmt.Sprintf("%ss", mhcSettings.NodeNotReadyTimeout), // With the 'second' suffix }); err != nil { return "", fmt.Errorf("could not generate a correct Memory Health Check YAML: %s", err) } @@ -150,7 +150,7 @@ func generateCapiYaml(clusterDetails *cseClusterCreationGoTemplateArguments) (st return "", err } - memoryHealthCheckYaml, err := generateMemoryHealthCheckYaml(clusterDetails, clusterDetails.Name) + memoryHealthCheckYaml, err := generateMemoryHealthCheckYaml(clusterDetails.MachineHealthCheck, clusterDetails.CseVersion, clusterDetails.Name) if err != nil { return "", err } diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 7b04c2338..bdafe33c8 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -268,7 +268,7 @@ func (input *CseClusterCreateInput) toCseClusterCreationGoTemplateContents(org * } } - mhc, err := getMachineHealthCheck(org.client, input.CseVersion, input.NodeHealthCheck) + mhc, err := getMachineHealthCheck(org.client, supportedCseVersions[input.CseVersion][0], input.NodeHealthCheck) if err != nil { return nil, err } @@ -339,7 +339,7 @@ func getTkgVersionBundleFromVAppTemplateName(ovaName string) (tkgVersionBundle, return result, err } - versionsMap := map[string]interface{}{} + versionsMap := map[string]any{} err = json.Unmarshal(cseTkgVersionsJson, &versionsMap) if err != nil { return result, err @@ -352,28 +352,27 @@ func getTkgVersionBundleFromVAppTemplateName(ovaName string) (tkgVersionBundle, // The map checking above guarantees that all splits and replaces will work result.KubernetesVersion = strings.Split(parsedOvaName, "-")[0] result.TkrVersion = strings.ReplaceAll(strings.Split(parsedOvaName, "-")[0], "+", "---") + "-" + strings.Split(parsedOvaName, "-")[1] - result.TkgVersion = versionMap.(map[string]interface{})["tkg"].(string) - result.EtcdVersion = versionMap.(map[string]interface{})["etcd"].(string) - result.CoreDnsVersion = versionMap.(map[string]interface{})["coreDns"].(string) + result.TkgVersion = versionMap.(map[string]any)["tkg"].(string) + result.EtcdVersion = versionMap.(map[string]any)["etcd"].(string) + result.CoreDnsVersion = versionMap.(map[string]any)["coreDns"].(string) return result, nil } // getMachineHealthCheck gets the required information from the CSE Server configuration RDE -func getMachineHealthCheck(client *Client, cseVersion string, isNodeHealthCheckActive bool) (*machineHealthCheck, error) { +func getMachineHealthCheck(client *Client, vcdKeConfigVersion string, isNodeHealthCheckActive bool) (*machineHealthCheck, error) { if !isNodeHealthCheckActive { return nil, nil } - currentCseVersion := supportedCseVersions[cseVersion] - rdes, err := getRdesByName(client, "vmware", "VCDKEConfig", currentCseVersion[0], "vcdKeConfig") + rdes, err := getRdesByName(client, "vmware", "VCDKEConfig", vcdKeConfigVersion, "vcdKeConfig") if err != nil { - return nil, fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", currentCseVersion[0], err) + return nil, fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", vcdKeConfigVersion, err) } if len(rdes) != 1 { return nil, fmt.Errorf("expected exactly one VCDKEConfig RDE but got %d", len(rdes)) } // TODO: Get the struct Type for this one - profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]interface{}) + profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]any) if !ok { return nil, fmt.Errorf("wrong format of VCDKEConfig, expected a 'profiles' array") } @@ -381,9 +380,11 @@ func getMachineHealthCheck(client *Client, cseVersion string, isNodeHealthCheckA return nil, fmt.Errorf("wrong format of VCDKEConfig, expected a single 'profiles' element, got %d", len(profiles)) } - // TODO: Get the struct Type for this one + mhc, ok := profiles[0].(map[string]any)["K8Config"].(map[string]any)["mhc"].(map[string]any) + if !ok { + return nil, nil + } result := machineHealthCheck{} - mhc := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"].(map[string]interface{}) result.MaxUnhealthyNodesPercentage = mhc["maxUnhealthyNodes"].(float64) result.NodeStartupTimeout = mhc["nodeStartupTimeout"].(string) result.NodeNotReadyTimeout = mhc["nodeUnknownTimeout"].(string) @@ -403,7 +404,7 @@ func getContainerRegistryUrl(client *Client, cseVersion string) (string, error) return "", fmt.Errorf("expected exactly one VCDKEConfig RDE but got %d", len(rdes)) } // TODO: Get the struct Type for this one - profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]interface{}) + profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]any) if !ok { return "", fmt.Errorf("wrong format of VCDKEConfig, expected a 'profiles' array") } @@ -411,7 +412,7 @@ func getContainerRegistryUrl(client *Client, cseVersion string) (string, error) return "", fmt.Errorf("wrong format of VCDKEConfig, expected a single 'profiles' element, got %d", len(profiles)) } // TODO: Check airgapped environments: https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.1.1a/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.1.1/GUID-F00BE796-B5F2-48F2-A012-546E2E694400.html - return fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"].(string)), nil + return fmt.Sprintf("%s/tkg", profiles[0].(map[string]any)["containerRegistryUrl"].(string)), nil } func getCseTemplate(cseVersion, templateName string) (string, error) { diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 1029fb14e..8a959a0be 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -142,8 +142,48 @@ func addWorkerPoolsYaml(docs []map[string]any, inputs []CseWorkerPoolCreateInput return nil } -func updateNodeHealthCheckYaml(docs []map[string]any, b bool) error { - return nil +func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName string, machineHealthCheck *machineHealthCheck) ([]map[string]any, error) { + mhcPosition := -1 + result := make([]map[string]any, len(yamlDocuments)) + for i, d := range yamlDocuments { + if d["kind"] == "MachineHealthCheck" { + mhcPosition = i + } + result[i] = d + } + + if mhcPosition < 0 { + // There is no MachineHealthCheck block + if machineHealthCheck == nil { + // We don't want it neither, so nothing to do + return result, nil + } + + // We need to add the block to the slice of YAML documents + mhcYaml, err := generateMemoryHealthCheckYaml(machineHealthCheck, "4.2", clusterName) + if err != nil { + return nil, err + } + var mhc map[string]any + err = yaml.Unmarshal([]byte(mhcYaml), &mhc) + if err != nil { + return nil, err + } + result = append(result, mhc) + } else { + // There is a MachineHealthCheck block + if machineHealthCheck != nil { + // We want it, but it is already there, so nothing to do + // TODO: What happens in UI if the VCDKEConfig MHC values are changed, does it get reflected in the cluster? + // If that's the case, we might need to update this value always + return result, nil + } + + // We don't want Machine Health Checks, we delete the YAML document + result[mhcPosition] = result[len(result)-1] // We override the MachineHealthCheck block with the last document + result = result[:len(result)-1] // We remove the last document (now duplicated) + } + return result, nil } // cseUpdateCapiYaml takes a CAPI YAML and modifies its Kubernetes template, its Control plane, its Worker pools @@ -199,7 +239,11 @@ func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateIn } if input.NodeHealthCheck != nil { - err := updateNodeHealthCheckYaml(yamlDocs, *input.NodeHealthCheck) + mhcSettings, err := getMachineHealthCheck(client, input.vcdKeConfigVersion, *input.NodeHealthCheck) + if err != nil { + return "", err + } + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, input.clusterName, mhcSettings) if err != nil { return "", err } diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 7b534072e..0a3904a51 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -211,6 +211,83 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { } } +// Test_cseUpdateNodeHealthCheckInYaml tests the update process of the Machine Health Check capabilities in a CAPI YAML. +func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { + capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") + if err != nil { + t.Fatalf("could not read CAPI YAML test file: %s", err) + } + + yamlDocs, err := unmarshalMultipleYamlDocuments(string(capiYaml)) + if err != nil { + t.Fatalf("could not unmarshal CAPI YAML test file: %s", err) + } + + clusterName := "" + for _, doc := range yamlDocs { + if doc["kind"] != "Cluster" { + continue + } + clusterName, err = traverseMapAndGet[string](doc, "metadata.name") + if err != nil { + t.Fatalf("incorrect CAPI YAML: %s", err) + } + } + if clusterName == "" { + t.Fatal("could not find the cluster name in the CAPI YAML test file") + } + + // Deactivates Machine Health Check + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, nil) + if err != nil { + t.Fatalf("%s", err) + } + + // The resulting documents should not have that document + for _, document := range yamlDocs { + if document["kind"] == "MachineHealthCheck" { + t.Fatal("Expected the MachineHealthCheck to be deleted, but it is there") + } + } + + // Enables Machine Health Check + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, &machineHealthCheck{ + MaxUnhealthyNodesPercentage: 12, + NodeStartupTimeout: "34", + NodeNotReadyTimeout: "56", + NodeUnknownTimeout: "78", + }) + if err != nil { + t.Fatalf("%s", err) + } + + // The resulting documents should have a MachineHealthCheck + found := false + for _, document := range yamlDocs { + if document["kind"] != "MachineHealthCheck" { + continue + } + maxUnhealthy, err := traverseMapAndGet[string](document, "spec.maxUnhealthy") + if err != nil { + t.Fatalf("%s", err) + } + if maxUnhealthy != "12%" { + t.Fatalf("expected a 'spec.maxUnhealthy' = 12%%, but got %s", maxUnhealthy) + } + nodeStartupTimeout, err := traverseMapAndGet[string](document, "spec.nodeStartupTimeout") + if err != nil { + t.Fatalf("%s", err) + } + if nodeStartupTimeout != "34s" { + t.Fatalf("expected a 'spec.nodeStartupTimeout' = 34s, but got %s", nodeStartupTimeout) + } + found = true + } + if !found { + t.Fatalf("expected a MachineHealthCheck block but got nothing") + } +} + // Test_unmarshalMultplieYamlDocuments tests the unmarshalling of multiple YAML documents with unmarshalMultplieYamlDocuments func Test_unmarshalMultplieYamlDocuments(t *testing.T) { capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") From eaf2846ed30015e63bcb46573e2f5befa076e0f1 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 1 Feb 2024 16:03:21 +0100 Subject: [PATCH 014/115] Update OVA fixed Signed-off-by: abarreiro --- govcd/cse_yaml.go | 64 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 8a959a0be..e4a89847e 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -54,21 +54,55 @@ func traverseMapAndGet[ResultType any](input interface{}, path string) (ResultTy // cseUpdateKubernetesTemplateInYaml updates the Kubernetes template OVA used by all the VCDMachineTemplate blocks func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]any, kubernetesTemplateOvaName string) error { - updated := false + tkgBundle, err := getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName) + if err != nil { + return err + } for _, d := range yamlDocuments { - if d["kind"] != "VCDMachineTemplate" { - continue - } + switch d["kind"] { + case "VCDMachineTemplate": + _, err := traverseMapAndGet[string](d, "spec.template.spec.template") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + d["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["template"] = kubernetesTemplateOvaName + case "MachineDeployment": + _, err := traverseMapAndGet[string](d, "spec.template.spec.version") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + d["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["version"] = tkgBundle.KubernetesVersion + case "Cluster": + _, err := traverseMapAndGet[string](d, "metadata.annotations.TKGVERSION") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + d["metadata"].(map[string]any)["annotations"].(map[string]any)["TKGVERSION"] = tkgBundle.TkgVersion - _, err := traverseMapAndGet[string](d, "spec.template.spec.template") - if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + _, err = traverseMapAndGet[string](d, "metadata.labels.tanzuKubernetesRelease") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + d["metadata"].(map[string]any)["labels"].(map[string]any)["tanzuKubernetesRelease"] = tkgBundle.TkrVersion + case "KubeadmControlPlane": + _, err := traverseMapAndGet[string](d, "spec.version") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + d["spec"].(map[string]any)["version"] = tkgBundle.KubernetesVersion + + _, err = traverseMapAndGet[string](d, "spec.kubeadmConfigSpec.clusterConfiguration.dns.imageTag") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + d["spec"].(map[string]any)["kubeadmConfigSpec"].(map[string]any)["clusterConfiguration"].(map[string]any)["dns"].(map[string]any)["imageTag"] = tkgBundle.CoreDnsVersion + + _, err = traverseMapAndGet[string](d, "spec.kubeadmConfigSpec.clusterConfiguration.etcd.local.imageTag") + if err != nil { + return fmt.Errorf("incorrect CAPI YAML: %s", err) + } + d["spec"].(map[string]any)["kubeadmConfigSpec"].(map[string]any)["clusterConfiguration"].(map[string]any)["etcd"].(map[string]any)["local"].(map[string]any)["imageTag"] = tkgBundle.EtcdVersion } - d["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["template"] = kubernetesTemplateOvaName - updated = true - } - if !updated { - return fmt.Errorf("could not find any template inside the VCDMachineTemplate blocks in the CAPI YAML") } return nil } @@ -138,8 +172,8 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[ return nil } -func addWorkerPoolsYaml(docs []map[string]any, inputs []CseWorkerPoolCreateInput) error { - return nil +func cseAddWorkerPoolsInYaml(docs []map[string]any, inputs []CseWorkerPoolCreateInput) ([]map[string]any, error) { + return nil, nil } func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName string, machineHealthCheck *machineHealthCheck) ([]map[string]any, error) { @@ -232,7 +266,7 @@ func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateIn } if input.NewWorkerPools != nil { - err := addWorkerPoolsYaml(yamlDocs, *input.NewWorkerPools) + yamlDocs, err = cseAddWorkerPoolsInYaml(yamlDocs, *input.NewWorkerPools) if err != nil { return capiYaml, err } From bc524cf79365b29950d6a22aa9b7941e7b89cf6e Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 1 Feb 2024 18:15:58 +0100 Subject: [PATCH 015/115] Refactor Signed-off-by: abarreiro --- govcd/cse.go | 260 +++++------------------------------ govcd/cse_template.go | 8 +- govcd/cse_test.go | 22 +-- govcd/cse_type.go | 197 +++++++++++++++++++++++++++ govcd/cse_util.go | 223 ++++++++++++++++++++---------- govcd/cse_yaml.go | 4 +- govcd/cse_yaml_unit_test.go | 30 ++++- types/v56/cse.go | 262 +++++++++++++++++------------------- 8 files changed, 552 insertions(+), 454 deletions(-) create mode 100644 govcd/cse_type.go diff --git a/govcd/cse.go b/govcd/cse.go index 992941464..ed3d545ba 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -1,7 +1,6 @@ package govcd import ( - "embed" "encoding/json" "fmt" "github.com/vmware/go-vcloud-director/v2/types/v56" @@ -10,122 +9,12 @@ import ( "time" ) -// supportedCseVersions is a map that contains only the supported CSE versions as keys, -// and its corresponding components versions as a slice of strings. The first string is the VCDKEConfig RDE Type version, -// then the CAPVCD RDE Type version and finally the CAPVCD Behavior version. -// TODO: Is this really necessary? What happens in UI if I have a 1.1.0-1.2.0-1.0.0 (4.2) cluster and then CSE is updated to 4.3? -var supportedCseVersions = map[string][]string{ - "4.2": { - "1.1.0", // VCDKEConfig RDE Type version - "1.2.0", // CAPVCD RDE Type version - "1.0.0", // CAPVCD Behavior version - }, -} - -// CseClusterApiProviderCluster is a type for handling ClusterApiProviderVCD (CAPVCD) cluster instances created -// by the Container Service Extension (CSE) -type CseClusterApiProviderCluster struct { - Capvcd *types.Capvcd - ID string - Owner string - Etag string - client *Client - - // TODO: Updated fields are inside the YAML file, like if you update the Control Plane replicas, you need to inspect - // the YAML to get the updated value. Inspecting Capvcd fields will do nothing. So I need to put here this information - // for convenience. - CseVersion string -} - -// CseClusterCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods -// must set in order to create a Kubernetes cluster. -type CseClusterCreateInput struct { - Name string - OrganizationId string - VdcId string - NetworkId string - KubernetesTemplateOvaId string - CseVersion string - ControlPlane CseControlPlaneCreateInput - WorkerPools []CseWorkerPoolCreateInput - DefaultStorageClass *CseDefaultStorageClassCreateInput // Optional - Owner string // Optional, if not set will pick the current user present in the VCDClient - ApiToken string - NodeHealthCheck bool - PodCidr string - ServiceCidr string - SshPublicKey string - VirtualIpSubnet string - AutoRepairOnErrors bool -} - -// CseControlPlaneCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods -// must set in order to specify the Control Plane inside a CseClusterCreateInput object. -type CseControlPlaneCreateInput struct { - MachineCount int - DiskSizeGi int - SizingPolicyId string // Optional - PlacementPolicyId string // Optional - StorageProfileId string // Optional - Ip string // Optional -} - -// CseWorkerPoolCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods -// must set in order to specify one Worker Pool inside a CseClusterCreateInput object. -type CseWorkerPoolCreateInput struct { - Name string - MachineCount int - DiskSizeGi int - SizingPolicyId string // Optional - PlacementPolicyId string // Optional - VGpuPolicyId string // Optional - StorageProfileId string // Optional -} - -// CseDefaultStorageClassCreateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods -// must set in order to specify a Default Storage Class inside a CseClusterCreateInput object. -type CseDefaultStorageClassCreateInput struct { - StorageProfileId string - Name string - ReclaimPolicy string - Filesystem string -} - -// CseClusterUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods -// must set in order to update a Kubernetes cluster. -type CseClusterUpdateInput struct { - KubernetesTemplateOvaId *string - ControlPlane *CseControlPlaneUpdateInput - WorkerPools *map[string]CseWorkerPoolUpdateInput // Maps a node pool name with its contents - NewWorkerPools *[]CseWorkerPoolCreateInput - NodeHealthCheck *bool - AutoRepairOnErrors *bool - - // Private fields that are computed, not requested to the consumer of this struct - vcdKeConfigVersion string - clusterName string -} - -// CseControlPlaneUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods -// must set in order to specify the Control Plane inside a CseClusterUpdateInput object. -type CseControlPlaneUpdateInput struct { - MachineCount int -} - -// CseWorkerPoolUpdateInput defines the required elements that the consumer of these Container Service Extension (CSE) methods -// must set in order to specify one Worker Pool inside a CseClusterCreateInput object. -type CseWorkerPoolUpdateInput struct { - MachineCount int -} - -//go:embed cse -var cseFiles embed.FS - -// CseCreateKubernetesCluster creates a Kubernetes cluster with the data given as input (CseClusterCreateInput). If the given -// timeout is 0, it waits forever for the cluster creation. Otherwise, if the timeout is reached and the cluster is not available, -// it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster in the returned CseClusterApiProviderCluster. -// If the cluster is created correctly, returns all the data in CseClusterApiProviderCluster. -func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterCreateInput, timeoutMinutes time.Duration) (*CseClusterApiProviderCluster, error) { +// CseCreateKubernetesCluster creates a Kubernetes cluster with the data given as input (CseClusterSettings). If the given +// timeout is 0, it waits forever for the cluster creation. Otherwise, if the timeout is reached and the cluster is not available +// (in "provisioned" state), it will return an error (the cluster will be left in VCD in any state) and the latest status +// of the cluster in the returned CseKubernetesCluster. +// If the cluster is created correctly, returns all the data in CseKubernetesCluster. +func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterSettings, timeoutMinutes time.Duration) (*CseKubernetesCluster, error) { clusterId, err := org.CseCreateKubernetesClusterAsync(clusterData) if err != nil { return nil, err @@ -139,11 +28,11 @@ func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterCreateInput, ti return cluster, nil } -// CseCreateKubernetesClusterAsync creates a Kubernetes cluster with the data given as input (CseClusterCreateInput), but does not +// CseCreateKubernetesClusterAsync creates a Kubernetes cluster with the data given as input (CseClusterSettings), but does not // wait for the creation process to finish, so it doesn't monitor for any errors during the process. It returns just the ID of -// the created cluster. One can manually check the status of the cluster with GetKubernetesClusterById and the result of this method. -func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterCreateInput) (string, error) { - goTemplateContents, err := clusterData.toCseClusterCreationGoTemplateContents(org) +// the created cluster. One can manually check the status of the cluster with Org.CseGetKubernetesClusterById and the result of this method. +func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) (string, error) { + goTemplateContents, err := cseClusterSettingsToInternal(clusterData, org) if err != nil { return "", err } @@ -169,7 +58,7 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterCreateInpu } // CseGetKubernetesClusterById retrieves a CSE Kubernetes cluster from VCD by its unique ID -func (org *Org) CseGetKubernetesClusterById(id string) (*CseClusterApiProviderCluster, error) { +func (org *Org) CseGetKubernetesClusterById(id string) (*CseKubernetesCluster, error) { rde, err := getRdeById(org.client, id) if err != nil { return nil, err @@ -178,54 +67,54 @@ func (org *Org) CseGetKubernetesClusterById(id string) (*CseClusterApiProviderCl if rde.DefinedEntity.Org.ID != org.Org.ID { return nil, fmt.Errorf("could not find any Kubernetes cluster with ID '%s' in Organization '%s': %s", id, org.Org.Name, ErrorEntityNotFound) } - return rde.cseConvertToCapvcdCluster() + return cseConvertToCseClusterApiProviderClusterType(rde) } // Refresh gets the latest information about the receiver cluster and updates its properties. -func (cluster *CseClusterApiProviderCluster) Refresh() error { +func (cluster *CseKubernetesCluster) Refresh() error { rde, err := getRdeById(cluster.client, cluster.ID) if err != nil { return err } - refreshed, err := rde.cseConvertToCapvcdCluster() + refreshed, err := cseConvertToCseClusterApiProviderClusterType(rde) if err != nil { return err } - cluster.Capvcd = refreshed.Capvcd + cluster.capvcdType = refreshed.capvcdType cluster.Etag = refreshed.Etag return nil } // UpdateWorkerPools executes an update on the receiver cluster to change the existing worker pools. -func (cluster *CseClusterApiProviderCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput) error { +func (cluster *CseKubernetesCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput) error { return cluster.Update(CseClusterUpdateInput{ WorkerPools: &input, }) } // AddWorkerPools executes an update on the receiver cluster to add new worker pools. -func (cluster *CseClusterApiProviderCluster) AddWorkerPools(input []CseWorkerPoolCreateInput) error { +func (cluster *CseKubernetesCluster) AddWorkerPools(input []CseWorkerPoolSettings) error { return cluster.Update(CseClusterUpdateInput{ NewWorkerPools: &input, }) } // UpdateControlPlane executes an update on the receiver cluster to change the existing control plane. -func (cluster *CseClusterApiProviderCluster) UpdateControlPlane(input CseControlPlaneUpdateInput) error { +func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpdateInput) error { return cluster.Update(CseClusterUpdateInput{ ControlPlane: &input, }) } // ChangeKubernetesTemplate executes an update on the receiver cluster to change the Kubernetes template of the cluster. -func (cluster *CseClusterApiProviderCluster) ChangeKubernetesTemplate(kubernetesTemplateOvaId string) error { +func (cluster *CseKubernetesCluster) ChangeKubernetesTemplate(kubernetesTemplateOvaId string) error { return cluster.Update(CseClusterUpdateInput{ KubernetesTemplateOvaId: &kubernetesTemplateOvaId, }) } // SetHealthCheck executes an update on the receiver cluster to enable or disable the machine health check capabilities. -func (cluster *CseClusterApiProviderCluster) SetHealthCheck(healthCheckEnabled bool) error { +func (cluster *CseKubernetesCluster) SetHealthCheck(healthCheckEnabled bool) error { return cluster.Update(CseClusterUpdateInput{ NodeHealthCheck: &healthCheckEnabled, }) @@ -233,7 +122,7 @@ func (cluster *CseClusterApiProviderCluster) SetHealthCheck(healthCheckEnabled b // SetAutoRepairOnErrors executes an update on the receiver cluster to change the flag that controls the auto-repair // capabilities of CSE. -func (cluster *CseClusterApiProviderCluster) SetAutoRepairOnErrors(autoRepairOnErrors bool) error { +func (cluster *CseKubernetesCluster) SetAutoRepairOnErrors(autoRepairOnErrors bool) error { return cluster.Update(CseClusterUpdateInput{ AutoRepairOnErrors: &autoRepairOnErrors, }) @@ -242,34 +131,34 @@ func (cluster *CseClusterApiProviderCluster) SetAutoRepairOnErrors(autoRepairOnE // Update executes a synchronous update on the receiver cluster to perform a update on any of the allowed parameters of the cluster. If the given // timeout is 0, it waits forever for the cluster update to finish. Otherwise, if the timeout is reached and the cluster is not available, // it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster will be available in the -// receiver CseClusterApiProviderCluster. -func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput) error { +// receiver CseKubernetesCluster. +func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput) error { err := cluster.Refresh() if err != nil { return err } - if cluster.Capvcd.Status.VcdKe.State == "" { + if cluster.capvcdType.Status.VcdKe.State == "" { return fmt.Errorf("can't update a Kubernetes cluster that does not have any state") } - if cluster.Capvcd.Status.VcdKe.State != "provisioned" { - return fmt.Errorf("can't update a Kubernetes cluster that is not in 'provisioned' state, as it is in '%s'", cluster.Capvcd.Status.VcdKe.State) + if cluster.capvcdType.Status.VcdKe.State != "provisioned" { + return fmt.Errorf("can't update a Kubernetes cluster that is not in 'provisioned' state, as it is in '%s'", cluster.capvcdType.Status.VcdKe.State) } if input.AutoRepairOnErrors != nil { - cluster.Capvcd.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors + cluster.capvcdType.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors } // Computed attributes that are required, such as the VcdKeConfig version - input.clusterName = cluster.Capvcd.Name - input.vcdKeConfigVersion = cluster.Capvcd.Status.VcdKe.VcdKeVersion - updatedCapiYaml, err := cseUpdateCapiYaml(cluster.client, cluster.Capvcd.Spec.CapiYaml, input) + input.clusterName = cluster.Name + input.vcdKeConfigVersion = cluster.capvcdType.Status.VcdKe.VcdKeVersion + updatedCapiYaml, err := cseUpdateCapiYaml(cluster.client, cluster.capvcdType.Spec.CapiYaml, input) if err != nil { return err } - cluster.Capvcd.Spec.CapiYaml = updatedCapiYaml + cluster.capvcdType.Spec.CapiYaml = updatedCapiYaml - marshaledPayload, err := json.Marshal(cluster.Capvcd) + marshaledPayload, err := json.Marshal(cluster.capvcdType) if err != nil { return err } @@ -324,7 +213,7 @@ func (cluster *CseClusterApiProviderCluster) Update(input CseClusterUpdateInput) // Delete deletes a CSE Kubernetes cluster, waiting the specified amount of minutes. If the timeout is reached, this method // returns an error, even if the cluster is already marked for deletion. -func (cluster *CseClusterApiProviderCluster) Delete(timeoutMinutes time.Duration) error { +func (cluster *CseKubernetesCluster) Delete(timeoutMinutes time.Duration) error { logHttpResponse := util.LogHttpResponse // The following loop is constantly polling VCD to retrieve the RDE, which has a big JSON inside, so we avoid filling @@ -383,86 +272,3 @@ func (cluster *CseClusterApiProviderCluster) Delete(timeoutMinutes time.Duration } return fmt.Errorf("timeout of %v minutes reached, the cluster was not marked for deletion, please try again", timeoutMinutes) } - -// cseConvertToCapvcdCluster takes the receiver, which is a generic RDE that must represent an existing CSE Kubernetes cluster, -// and transforms it to a specific Container Service Extension CAPVCD object that represents the same cluster, but -// it is easy to explore and consume. If the receiver object does not contain a CAPVCD object, this method -// will obviously return an error. -func (rde *DefinedEntity) cseConvertToCapvcdCluster() (*CseClusterApiProviderCluster, error) { - requiredType := "vmware:capvcdCluster" - - if !strings.Contains(rde.DefinedEntity.ID, requiredType) || !strings.Contains(rde.DefinedEntity.EntityType, requiredType) { - return nil, fmt.Errorf("the receiver RDE is not a '%s' entity, it is '%s'", requiredType, rde.DefinedEntity.EntityType) - } - - entityBytes, err := json.Marshal(rde.DefinedEntity.Entity) - if err != nil { - return nil, fmt.Errorf("could not marshal the RDE contents to create a Capvcd instance: %s", err) - } - - result := &CseClusterApiProviderCluster{ - Capvcd: &types.Capvcd{}, - ID: rde.DefinedEntity.ID, - Etag: rde.Etag, - client: rde.client, - } - if rde.DefinedEntity.Owner != nil { - result.Owner = rde.DefinedEntity.Owner.Name - } - - err = json.Unmarshal(entityBytes, result.Capvcd) - if err != nil { - return nil, fmt.Errorf("could not unmarshal the RDE contents to create a Capvcd instance: %s", err) - } - return result, nil -} - -// waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if timeoutMinutes = 0) -// or until this timeout is reached. If the cluster is in "provisioned" state before the given timeout, it returns a CseClusterApiProviderCluster object -// representing the Kubernetes cluster with all the latest information. -// If one of the states of the cluster at a given point is "error", this function also checks whether the cluster has the "Auto Repair on Errors" flag enabled, -// so it keeps waiting if it's true. -// If timeout is reached before the cluster, it returns an error. -func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinutes time.Duration) (*CseClusterApiProviderCluster, error) { - var elapsed time.Duration - logHttpResponse := util.LogHttpResponse - sleepTime := 30 - - // The following loop is constantly polling VCD to retrieve the RDE, which has a big JSON inside, so we avoid filling - // the log with these big payloads. We use defer to be sure that we restore the initial logging state. - defer func() { - util.LogHttpResponse = logHttpResponse - }() - - start := time.Now() - var capvcdCluster *CseClusterApiProviderCluster - for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies timeoutMinutes=0, we wait forever - util.LogHttpResponse = false - rde, err := getRdeById(client, clusterId) - util.LogHttpResponse = logHttpResponse - if err != nil { - return nil, err - } - - capvcdCluster, err = rde.cseConvertToCapvcdCluster() - if err != nil { - return nil, err - } - - switch capvcdCluster.Capvcd.Status.VcdKe.State { - case "provisioned": - return capvcdCluster, nil - case "error": - // We just finish if auto-recovery is disabled, otherwise we just let CSE fixing things in background - if !capvcdCluster.Capvcd.Spec.VcdKe.AutoRepairOnErrors { - // Try to give feedback about what went wrong, which is located in a set of events in the RDE payload - return capvcdCluster, fmt.Errorf("got an error and 'auto repair on errors' is disabled, aborting. Errors: %s", capvcdCluster.Capvcd.Status.Capvcd.ErrorSet[len(capvcdCluster.Capvcd.Status.Capvcd.ErrorSet)-1].AdditionalDetails.DetailedError) - } - } - - util.Logger.Printf("[DEBUG] Cluster '%s' is in '%s' state, will check again in %d seconds", capvcdCluster.ID, capvcdCluster.Capvcd.Status.VcdKe.State, sleepTime) - elapsed = time.Since(start) - time.Sleep(time.Duration(sleepTime) * time.Second) - } - return capvcdCluster, fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, capvcdCluster.Capvcd.Status.VcdKe.State) -} diff --git a/govcd/cse_template.go b/govcd/cse_template.go index f9db7670d..c58910445 100644 --- a/govcd/cse_template.go +++ b/govcd/cse_template.go @@ -14,7 +14,7 @@ import ( // getCseKubernetesClusterCreationPayload gets the payload for the RDE that will trigger a Kubernetes cluster creation. // It generates a valid YAML that is embedded inside the RDE JSON, then it is returned as an unmarshaled // generic map, that allows to be sent to VCD as it is. -func getCseKubernetesClusterCreationPayload(goTemplateContents *cseClusterCreationGoTemplateArguments) (map[string]interface{}, error) { +func getCseKubernetesClusterCreationPayload(goTemplateContents *cseClusterSettingsInternal) (map[string]interface{}, error) { capiYaml, err := generateCapiYaml(goTemplateContents) if err != nil { return nil, err @@ -60,7 +60,7 @@ func getCseKubernetesClusterCreationPayload(goTemplateContents *cseClusterCreati } // generateNodePoolYaml generates YAML blocks corresponding to the Kubernetes node pools. -func generateNodePoolYaml(clusterDetails *cseClusterCreationGoTemplateArguments) (string, error) { +func generateNodePoolYaml(clusterDetails *cseClusterSettingsInternal) (string, error) { workerPoolTmpl, err := getCseTemplate(clusterDetails.CseVersion, "capiyaml_workerpool") if err != nil { return "", err @@ -105,7 +105,7 @@ func generateNodePoolYaml(clusterDetails *cseClusterCreationGoTemplateArguments) } // generateMemoryHealthCheckYaml generates a YAML block corresponding to the Kubernetes memory health check. -func generateMemoryHealthCheckYaml(mhcSettings *machineHealthCheck, cseVersion, clusterName string) (string, error) { +func generateMemoryHealthCheckYaml(mhcSettings *cseMachineHealthCheckInternal, cseVersion, clusterName string) (string, error) { if mhcSettings == nil { return "", nil } @@ -135,7 +135,7 @@ func generateMemoryHealthCheckYaml(mhcSettings *machineHealthCheck, cseVersion, // generateCapiYaml generates the YAML string that is required during Kubernetes cluster creation, to be embedded // in the CAPVCD cluster JSON payload. This function picks data from the Terraform schema and the createClusterDto to // populate several Go templates and build a final YAML. -func generateCapiYaml(clusterDetails *cseClusterCreationGoTemplateArguments) (string, error) { +func generateCapiYaml(clusterDetails *cseClusterSettingsInternal) (string, error) { clusterTmpl, err := getCseTemplate(clusterDetails.CseVersion, "capiyaml_cluster") if err != nil { return "", err diff --git a/govcd/cse_test.go b/govcd/cse_test.go index f6b0cd509..a4b96ad1f 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -66,28 +66,28 @@ func (vcd *TestVCD) Test_Cse(check *C) { workerPoolName := "worker-pool-1" - cluster, err := org.CseCreateKubernetesCluster(CseClusterCreateInput{ + cluster, err := org.CseCreateKubernetesCluster(CseClusterSettings{ Name: "test-cse", OrganizationId: org.Org.ID, VdcId: vdc.Vdc.ID, NetworkId: net.OrgVDCNetwork.ID, KubernetesTemplateOvaId: ova.VAppTemplate.ID, CseVersion: "4.2", - ControlPlane: CseControlPlaneCreateInput{ + ControlPlane: CseControlPlaneSettings{ MachineCount: 1, DiskSizeGi: 20, SizingPolicyId: policies[0].VdcComputePolicyV2.ID, StorageProfileId: sp.ID, Ip: "", }, - WorkerPools: []CseWorkerPoolCreateInput{{ + WorkerPools: []CseWorkerPoolSettings{{ Name: workerPoolName, MachineCount: 1, DiskSizeGi: 20, SizingPolicyId: policies[0].VdcComputePolicyV2.ID, StorageProfileId: sp.ID, }}, - DefaultStorageClass: &CseDefaultStorageClassCreateInput{ + DefaultStorageClass: &CseDefaultStorageClassSettings{ StorageProfileId: sp.ID, Name: "storage-class-1", ReclaimPolicy: "delete", @@ -103,7 +103,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(err, IsNil) check.Assert(cluster.ID, Not(Equals), "") check.Assert(cluster.Etag, Not(Equals), "") - check.Assert(cluster.Capvcd.Status.VcdKe.State, Equals, "provisioned") + check.Assert(cluster.capvcdType.Status.VcdKe.State, Equals, "provisioned") err = cluster.Refresh() check.Assert(err, IsNil) @@ -111,14 +111,14 @@ func (vcd *TestVCD) Test_Cse(check *C) { clusterGet, err := org.CseGetKubernetesClusterById(cluster.ID) check.Assert(err, IsNil) check.Assert(cluster.ID, Equals, clusterGet.ID) - check.Assert(cluster.Capvcd.Name, Equals, clusterGet.Capvcd.Name) + check.Assert(cluster.Name, Equals, clusterGet.Name) check.Assert(cluster.Owner, Equals, clusterGet.Owner) - check.Assert(cluster.Capvcd.Metadata, DeepEquals, clusterGet.Capvcd.Metadata) - check.Assert(cluster.Capvcd.Spec.VcdKe, DeepEquals, clusterGet.Capvcd.Spec.VcdKe) + check.Assert(cluster.capvcdType.Metadata, DeepEquals, clusterGet.capvcdType.Metadata) + check.Assert(cluster.capvcdType.Spec.VcdKe, DeepEquals, clusterGet.capvcdType.Spec.VcdKe) // Update worker pool from 1 node to 2 // Pre-check. This should be 1, as it was created with just 1 pool - for _, nodePool := range cluster.Capvcd.Status.Capvcd.NodePool { + for _, nodePool := range cluster.capvcdType.Status.Capvcd.NodePool { if nodePool.Name == workerPoolName { check.Assert(nodePool.DesiredReplicas, Equals, 1) } @@ -129,7 +129,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { // Post-check. This should be 2, as it should have scaled up foundWorkerPool := false - for _, nodePool := range cluster.Capvcd.Status.Capvcd.NodePool { + for _, nodePool := range cluster.capvcdType.Status.Capvcd.NodePool { if nodePool.Name == workerPoolName { foundWorkerPool = true check.Assert(nodePool.DesiredReplicas, Equals, 2) @@ -160,7 +160,7 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { // Post-check. This should be 2, as it should have scaled up foundWorkerPool := false - for _, nodePool := range cluster.Capvcd.Status.Capvcd.NodePool { + for _, nodePool := range cluster.capvcdType.Status.Capvcd.NodePool { if nodePool.Name == workerPoolName { foundWorkerPool = true check.Assert(nodePool.DesiredReplicas, Equals, 2) diff --git a/govcd/cse_type.go b/govcd/cse_type.go new file mode 100644 index 000000000..be795e22f --- /dev/null +++ b/govcd/cse_type.go @@ -0,0 +1,197 @@ +package govcd + +import ( + "embed" + "github.com/vmware/go-vcloud-director/v2/types/v56" + "time" +) + +// CseKubernetesCluster is a type for handling a Kubernetes cluster created by the Container Service Extension (CSE) +type CseKubernetesCluster struct { + CseClusterSettings + ID string + Etag string + KubernetesVersion string + TkgVersion string + CapvcdVersion string + ClusterResourceSetBindings []string + CpiVersion string + CsiVersion string + State string + Kubeconfig string + Events []CseClusterEvent + + client *Client + capvcdType *types.Capvcd +} + +// CseClusterEvent is an event that has occurred during the lifetime of a Container Service Extension (CSE) Kubernetes cluster. +type CseClusterEvent struct { + Name string + OccurredAt time.Time + Details string +} + +// CseClusterSettings defines the required configuration of a Container Service Extension (CSE) Kubernetes cluster. +type CseClusterSettings struct { + Name string + OrganizationId string + VdcId string + NetworkId string + KubernetesTemplateOvaId string + CseVersion string + ControlPlane CseControlPlaneSettings + WorkerPools []CseWorkerPoolSettings + DefaultStorageClass *CseDefaultStorageClassSettings // Optional + Owner string // Optional, if not set will pick the current session user from the VCDClient + ApiToken string + NodeHealthCheck bool + PodCidr string + ServiceCidr string + SshPublicKey string + VirtualIpSubnet string + AutoRepairOnErrors bool +} + +// CseControlPlaneSettings defines the required configuration of a Control Plane of a Container Service Extension (CSE) Kubernetes cluster. +type CseControlPlaneSettings struct { + MachineCount int + DiskSizeGi int + SizingPolicyId string // Optional + PlacementPolicyId string // Optional + StorageProfileId string // Optional + Ip string // Optional +} + +// CseWorkerPoolSettings defines the required configuration of a Worker Pool of a Container Service Extension (CSE) Kubernetes cluster. +type CseWorkerPoolSettings struct { + Name string + MachineCount int + DiskSizeGi int + SizingPolicyId string // Optional + PlacementPolicyId string // Optional + VGpuPolicyId string // Optional + StorageProfileId string // Optional +} + +// CseDefaultStorageClassSettings defines the required configuration of a Default Storage Class of a Container Service Extension (CSE) Kubernetes cluster. +type CseDefaultStorageClassSettings struct { + StorageProfileId string + Name string + ReclaimPolicy string + Filesystem string +} + +// CseClusterUpdateInput defines the required configuration that a Container Service Extension (CSE) Kubernetes cluster needs in order to be updated. +type CseClusterUpdateInput struct { + KubernetesTemplateOvaId *string + ControlPlane *CseControlPlaneUpdateInput + WorkerPools *map[string]CseWorkerPoolUpdateInput // Maps a node pool name with its contents + NewWorkerPools *[]CseWorkerPoolSettings + NodeHealthCheck *bool + AutoRepairOnErrors *bool + + // Private fields that are computed, not requested to the consumer of this struct + vcdKeConfigVersion string + clusterName string +} + +// CseControlPlaneUpdateInput defines the required configuration that the Control Plane of the Container Service Extension (CSE) Kubernetes cluster +// needs in order to be updated. +type CseControlPlaneUpdateInput struct { + MachineCount int +} + +// CseWorkerPoolUpdateInput defines the required configuration that a Worker Pool of the Container Service Extension (CSE) Kubernetes cluster +// needs in order to be updated. +type CseWorkerPoolUpdateInput struct { + MachineCount int +} + +// cseClusterSettingsInternal defines the required arguments that are required by the CSE Server used internally to specify +// a Kubernetes cluster. These are not set by the user, but instead they are computed from a valid +// CseClusterSettings object in the cseClusterSettingsToInternal method. These fields are then +// inserted in Go templates to render a final JSON that is valid to be used as the cluster Runtime Defined Entity (RDE) payload. +// +// The main difference between CseClusterSettings and this structure is that the first one uses IDs and this one uses names, among +// other differences like the computed TkgVersionBundle. +type cseClusterSettingsInternal struct { + CseVersion string + Name string + OrganizationName string + VdcName string + NetworkName string + KubernetesTemplateOvaName string + TkgVersionBundle tkgVersionBundle + CatalogName string + RdeType *types.DefinedEntityType + ControlPlane cseControlPlaneSettingsInternal + WorkerPools []cseWorkerPoolSettingsInternal + DefaultStorageClass cseDefaultStorageClassInternal + MachineHealthCheck *cseMachineHealthCheckInternal + Owner string + ApiToken string + VcdUrl string + ContainerRegistryUrl string + VirtualIpSubnet string + SshPublicKey string + PodCidr string + ServiceCidr string + AutoRepairOnErrors bool +} + +// cseControlPlaneSettingsInternal defines the Control Plane inside cseClusterSettingsInternal +type cseControlPlaneSettingsInternal struct { + MachineCount int + DiskSizeGi int + SizingPolicyName string + PlacementPolicyName string + StorageProfileName string + Ip string +} + +// cseWorkerPoolSettingsInternal defines a Worker Pool inside cseClusterSettingsInternal +type cseWorkerPoolSettingsInternal struct { + Name string + MachineCount int + DiskSizeGi int + SizingPolicyName string + PlacementPolicyName string + VGpuPolicyName string + StorageProfileName string +} + +// cseDefaultStorageClassInternal defines a Default Storage Class inside cseClusterSettingsInternal +type cseDefaultStorageClassInternal struct { + StorageProfileName string + Name string + UseDeleteReclaimPolicy bool + Filesystem string +} + +// cseMachineHealthCheckInternal is a type that contains only the required and relevant fields from the VCDKEConfig (CSE Server) configuration, +// such as the Machine Health Check settings. +type cseMachineHealthCheckInternal struct { + MaxUnhealthyNodesPercentage float64 + NodeStartupTimeout string + NodeNotReadyTimeout string + NodeUnknownTimeout string +} + +// This collection of files contains all the Go Templates and resources required for the Container Service Extension (CSE) methods +// to work. +// +//go:embed cse +var cseFiles embed.FS + +// supportedCseVersions is a map that contains only the supported CSE versions as keys, +// and its corresponding components versions as a slice of strings. The first string is the VCDKEConfig RDE Type version, +// then the CAPVCD RDE Type version and finally the CAPVCD Behavior version. +// TODO: Is this really necessary? What happens in UI if I have a 1.1.0-1.2.0-1.0.0 (4.2) cluster and then CSE is updated to 4.3? +var supportedCseVersions = map[string][]string{ + "4.2": { + "1.1.0", // VCDKEConfig RDE Type version + "1.2.0", // CAPVCD RDE Type version + "1.0.0", // CAPVCD Behavior version + }, +} diff --git a/govcd/cse_util.go b/govcd/cse_util.go index bdafe33c8..c5f512df1 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -5,79 +5,160 @@ import ( "encoding/json" "fmt" "github.com/vmware/go-vcloud-director/v2/types/v56" + "github.com/vmware/go-vcloud-director/v2/util" "regexp" "strings" + "time" ) -// cseClusterCreationGoTemplateArguments defines the required arguments that are required by the Go templates used internally to specify -// a Kubernetes cluster. These are not set by the user, but instead they are computed from a valid -// CseClusterCreateInput object in the CseClusterCreateInput.toCseClusterCreationGoTemplateContents method. These fields are then -// inserted in Go templates to render a final JSON that is valid to be used as the cluster Runtime Defined Entity (RDE) payload. -type cseClusterCreationGoTemplateArguments struct { - CseVersion string - Name string - OrganizationName string - VdcName string - NetworkName string - KubernetesTemplateOvaName string - TkgVersionBundle tkgVersionBundle - CatalogName string - RdeType *types.DefinedEntityType - ControlPlane controlPlane - WorkerPools []workerPool - DefaultStorageClass defaultStorageClass - MachineHealthCheck *machineHealthCheck - Owner string - ApiToken string - VcdUrl string - ContainerRegistryUrl string - VirtualIpSubnet string - SshPublicKey string - PodCidr string - ServiceCidr string - AutoRepairOnErrors bool -} +// cseConvertToCseClusterApiProviderClusterType takes a generic RDE that must represent an existing CSE Kubernetes cluster, +// and transforms it to a specific Container Service Extension CseKubernetesCluster object that represents the same cluster, but +// it is easy to explore and consume. If the receiver object does not contain a CAPVCD object, this method +// will obviously return an error. +func cseConvertToCseClusterApiProviderClusterType(rde *DefinedEntity) (*CseKubernetesCluster, error) { + requiredType := "vmware:capvcdCluster" -// controlPlane defines the Control Plane inside cseClusterCreationGoTemplateArguments -type controlPlane struct { - MachineCount int - DiskSizeGi int - SizingPolicyName string - PlacementPolicyName string - StorageProfileName string - Ip string -} + if !strings.Contains(rde.DefinedEntity.ID, requiredType) || !strings.Contains(rde.DefinedEntity.EntityType, requiredType) { + return nil, fmt.Errorf("the receiver RDE is not a '%s' entity, it is '%s'", requiredType, rde.DefinedEntity.EntityType) + } -// workerPool defines a Worker pool inside cseClusterCreationGoTemplateArguments -type workerPool struct { - Name string - MachineCount int - DiskSizeGi int - SizingPolicyName string - PlacementPolicyName string - VGpuPolicyName string - StorageProfileName string -} + entityBytes, err := json.Marshal(rde.DefinedEntity.Entity) + if err != nil { + return nil, fmt.Errorf("could not marshal the RDE contents to create a capvcdType instance: %s", err) + } + + capvcd := &types.Capvcd{} + err = json.Unmarshal(entityBytes, &capvcd) + if err != nil { + return nil, fmt.Errorf("could not unmarshal the RDE contents to create a Capvcd instance: %s", err) + } + + result := &CseKubernetesCluster{ + CseClusterSettings: CseClusterSettings{ + Name: rde.DefinedEntity.Name, + ApiToken: "******", // We can't return this one, we return the "standard" 6-asterisk value + AutoRepairOnErrors: capvcd.Spec.VcdKe.AutoRepairOnErrors, + }, + ID: rde.DefinedEntity.ID, + Etag: rde.Etag, + KubernetesVersion: capvcd.Status.Capvcd.Upgrade.Current.KubernetesVersion, + TkgVersion: capvcd.Status.Capvcd.Upgrade.Current.TkgVersion, + CapvcdVersion: capvcd.Status.Capvcd.CapvcdVersion, + ClusterResourceSetBindings: make([]string, len(capvcd.Status.Capvcd.ClusterResourceSetBindings)), + CpiVersion: capvcd.Status.Cpi.Version, + CsiVersion: capvcd.Status.Csi.Version, + State: capvcd.Status.VcdKe.State, + client: rde.client, + capvcdType: capvcd, + } + for i, binding := range capvcd.Status.Capvcd.ClusterResourceSetBindings { + result.ClusterResourceSetBindings[i] = binding.ClusterResourceSetName + } + + if len(result.capvcdType.Status.Capvcd.VcdProperties.Organizations) == 0 { + return nil, fmt.Errorf("could not read Organizations from Capvcd type") + } + result.OrganizationId = result.capvcdType.Status.Capvcd.VcdProperties.Organizations[0].Id + if len(result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs) == 0 { + return nil, fmt.Errorf("could not read Org VDC Network from Capvcd type") + } + result.VdcId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Id + result.NetworkId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName // TODO: ID + + if rde.DefinedEntity.Owner == nil { + return nil, fmt.Errorf("could not read Owner from RDE") + } + result.Owner = rde.DefinedEntity.Owner.Name + + if result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.K8SStorageClassName != "" { + result.DefaultStorageClass = &CseDefaultStorageClassSettings{ + StorageProfileId: "", // TODO: ID + Name: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.K8SStorageClassName, + ReclaimPolicy: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName, + Filesystem: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName, + } + } + + yamlDocuments, err := unmarshalMultipleYamlDocuments(result.capvcdType.Spec.CapiYaml) + if err != nil { + return nil, err + } + + result.KubernetesTemplateOvaId = "" // TODO: YAML > ID + result.CseVersion = "" // TODO: Get opposite from supportedVersionsMap + // TODO: YAML > Control Plane + // TODO: YAML > Worker pools + // TODO: YAML > Health check + // YAML PodCidr: "", + // YAML ServiceCidr: "", + // YAML SshPublicKey: "", + // YAML VirtualIpSubnet: "", + + for _, yamlDocument := range yamlDocuments { + switch yamlDocument["kind"] { -// defaultStorageClass defines a Default Storage Class inside cseClusterCreationGoTemplateArguments -type defaultStorageClass struct { - StorageProfileName string - Name string - UseDeleteReclaimPolicy bool - Filesystem string + } + } + if err != nil { + return nil, fmt.Errorf("could not get the cluster state from the RDE contents: %s", err) + } + + return result, nil } -// machineHealthCheck is a type that contains only the required and relevant fields from the Container Service Extension (CSE) installation configuration, -// such as the Machine Health Check settings or the container registry URL. -type machineHealthCheck struct { - MaxUnhealthyNodesPercentage float64 - NodeStartupTimeout string - NodeNotReadyTimeout string - NodeUnknownTimeout string +// waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if timeoutMinutes = 0) +// or until this timeout is reached. If the cluster is in "provisioned" state before the given timeout, it returns a CseKubernetesCluster object +// representing the Kubernetes cluster with all the latest information. +// If one of the states of the cluster at a given point is "error", this function also checks whether the cluster has the "Auto Repair on Errors" flag enabled, +// so it keeps waiting if it's true. +// If timeout is reached before the cluster, it returns an error. +func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinutes time.Duration) (*CseKubernetesCluster, error) { + var elapsed time.Duration + logHttpResponse := util.LogHttpResponse + sleepTime := 30 + + // The following loop is constantly polling VCD to retrieve the RDE, which has a big JSON inside, so we avoid filling + // the log with these big payloads. We use defer to be sure that we restore the initial logging state. + defer func() { + util.LogHttpResponse = logHttpResponse + }() + + start := time.Now() + var capvcdCluster *CseKubernetesCluster + for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies timeoutMinutes=0, we wait forever + util.LogHttpResponse = false + rde, err := getRdeById(client, clusterId) + util.LogHttpResponse = logHttpResponse + if err != nil { + return nil, err + } + + capvcdCluster, err = cseConvertToCseClusterApiProviderClusterType(rde) + if err != nil { + return nil, err + } + + switch capvcdCluster.capvcdType.Status.VcdKe.State { + case "provisioned": + return capvcdCluster, nil + case "error": + // We just finish if auto-recovery is disabled, otherwise we just let CSE fixing things in background + if !capvcdCluster.capvcdType.Spec.VcdKe.AutoRepairOnErrors { + // Try to give feedback about what went wrong, which is located in a set of events in the RDE payload + return capvcdCluster, fmt.Errorf("got an error and 'auto repair on errors' is disabled, aborting") + // TODO return capvcdCluster, fmt.Errorf("got an error and 'auto repair on errors' is disabled, aborting. Errors: %s", capvcdCluster.capvcdType.Status.Capvcd.ErrorSet[len(capvcdCluster.capvcdType.Status.Capvcd.ErrorSet)-1].AdditionalDetails.DetailedError) + } + } + + util.Logger.Printf("[DEBUG] Cluster '%s' is in '%s' state, will check again in %d seconds", capvcdCluster.ID, capvcdCluster.capvcdType.Status.VcdKe.State, sleepTime) + elapsed = time.Since(start) + time.Sleep(time.Duration(sleepTime) * time.Second) + } + return capvcdCluster, fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, capvcdCluster.capvcdType.Status.VcdKe.State) } // validate validates the CSE Kubernetes cluster creation input data. Returns an error if some of the fields is wrong. -func (ccd *CseClusterCreateInput) validate() error { +func (ccd *CseClusterSettings) validate() error { cseNamesRegex, err := regexp.Compile(`^[a-z](?:[a-z0-9-]{0,29}[a-z0-9])?$`) if err != nil { return fmt.Errorf("could not compile regular expression '%s'", err) @@ -149,9 +230,9 @@ func (ccd *CseClusterCreateInput) validate() error { return nil } -// toCseClusterCreationGoTemplateContents transforms user input data (receiver CseClusterCreateInput) into the final payload that -// will be used to render the Go templates that define a Kubernetes cluster creation payload (cseClusterCreationGoTemplateArguments). -func (input *CseClusterCreateInput) toCseClusterCreationGoTemplateContents(org *Org) (*cseClusterCreationGoTemplateArguments, error) { +// cseClusterSettingsToInternal transforms user input data (CseClusterSettings) into the final payload that +// will be used to render the Go templates that define a Kubernetes cluster creation payload (cseClusterSettingsInternal). +func cseClusterSettingsToInternal(input CseClusterSettings, org *Org) (*cseClusterSettingsInternal, error) { err := input.validate() if err != nil { return nil, err @@ -161,7 +242,7 @@ func (input *CseClusterCreateInput) toCseClusterCreationGoTemplateContents(org * return nil, fmt.Errorf("cannot manipulate the CSE Kubernetes cluster creation input, the Organization is nil") } - output := &cseClusterCreationGoTemplateArguments{} + output := &cseClusterSettingsInternal{} output.OrganizationName = org.Org.Name vdc, err := org.GetVDCById(input.VdcId, true) @@ -235,9 +316,9 @@ func (input *CseClusterCreateInput) toCseClusterCreationGoTemplateContents(org * } // Now that everything is cached in memory, we can build the Node pools and Storage Class payloads - output.WorkerPools = make([]workerPool, len(input.WorkerPools)) + output.WorkerPools = make([]cseWorkerPoolSettingsInternal, len(input.WorkerPools)) for i, w := range input.WorkerPools { - output.WorkerPools[i] = workerPool{ + output.WorkerPools[i] = cseWorkerPoolSettingsInternal{ Name: w.Name, MachineCount: w.MachineCount, DiskSizeGi: w.DiskSizeGi, @@ -247,7 +328,7 @@ func (input *CseClusterCreateInput) toCseClusterCreationGoTemplateContents(org * output.WorkerPools[i].VGpuPolicyName = idToNameCache[w.VGpuPolicyId] output.WorkerPools[i].StorageProfileName = idToNameCache[w.StorageProfileId] } - output.ControlPlane = controlPlane{ + output.ControlPlane = cseControlPlaneSettingsInternal{ MachineCount: input.ControlPlane.MachineCount, DiskSizeGi: input.ControlPlane.DiskSizeGi, SizingPolicyName: idToNameCache[input.ControlPlane.SizingPolicyId], @@ -257,7 +338,7 @@ func (input *CseClusterCreateInput) toCseClusterCreationGoTemplateContents(org * } if input.DefaultStorageClass != nil { - output.DefaultStorageClass = defaultStorageClass{ + output.DefaultStorageClass = cseDefaultStorageClassInternal{ StorageProfileName: idToNameCache[input.DefaultStorageClass.StorageProfileId], Name: input.DefaultStorageClass.Name, Filesystem: input.DefaultStorageClass.Filesystem, @@ -359,7 +440,7 @@ func getTkgVersionBundleFromVAppTemplateName(ovaName string) (tkgVersionBundle, } // getMachineHealthCheck gets the required information from the CSE Server configuration RDE -func getMachineHealthCheck(client *Client, vcdKeConfigVersion string, isNodeHealthCheckActive bool) (*machineHealthCheck, error) { +func getMachineHealthCheck(client *Client, vcdKeConfigVersion string, isNodeHealthCheckActive bool) (*cseMachineHealthCheckInternal, error) { if !isNodeHealthCheckActive { return nil, nil } @@ -384,7 +465,7 @@ func getMachineHealthCheck(client *Client, vcdKeConfigVersion string, isNodeHeal if !ok { return nil, nil } - result := machineHealthCheck{} + result := cseMachineHealthCheckInternal{} result.MaxUnhealthyNodesPercentage = mhc["maxUnhealthyNodes"].(float64) result.NodeStartupTimeout = mhc["nodeStartupTimeout"].(string) result.NodeNotReadyTimeout = mhc["nodeUnknownTimeout"].(string) diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index e4a89847e..dfcb3ee39 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -172,11 +172,11 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[ return nil } -func cseAddWorkerPoolsInYaml(docs []map[string]any, inputs []CseWorkerPoolCreateInput) ([]map[string]any, error) { +func cseAddWorkerPoolsInYaml(docs []map[string]any, inputs []CseWorkerPoolSettings) ([]map[string]any, error) { return nil, nil } -func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName string, machineHealthCheck *machineHealthCheck) ([]map[string]any, error) { +func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName string, machineHealthCheck *cseMachineHealthCheckInternal) ([]map[string]any, error) { mhcPosition := -1 result := make([]map[string]any, len(yamlDocuments)) for i, d := range yamlDocuments { diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 0a3904a51..b031228b0 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -38,9 +38,18 @@ func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { if oldOvaName == "" { t.Fatalf("the OVA that needs to be changed is empty") } + oldTkgBundle, err := getTkgVersionBundleFromVAppTemplateName(oldOvaName) + if err != nil { + t.Fatalf("%s", err) + } // We call the function to update the old OVA with the new one - newOvaName := "my-super-ova-name" + newOvaName := "ubuntu-2004-kube-v1.19.16+vmware.1-tkg.2-fba68db15591c15fcd5f26b512663a42" + newTkgBundle, err := getTkgVersionBundleFromVAppTemplateName(newOvaName) + if err != nil { + t.Fatalf("%s", err) + } + err = cseUpdateKubernetesTemplateInYaml(yamlDocs, newOvaName) if err != nil { t.Fatalf("%s", err) @@ -53,7 +62,22 @@ func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { // No document should have the old OVA if !strings.Contains(updatedYaml, newOvaName) || strings.Contains(updatedYaml, oldOvaName) { - t.Fatalf("failed updating the Kubernetes OVA template in the Control Plane:\n%s", updatedYaml) + t.Fatalf("failed updating the Kubernetes OVA template:\n%s", updatedYaml) + } + if !strings.Contains(updatedYaml, newTkgBundle.KubernetesVersion) || strings.Contains(updatedYaml, oldTkgBundle.KubernetesVersion) { + t.Fatalf("failed updating the Kubernetes version:\n%s", updatedYaml) + } + if !strings.Contains(updatedYaml, newTkgBundle.TkrVersion) || strings.Contains(updatedYaml, oldTkgBundle.TkrVersion) { + t.Fatalf("failed updating the Tanzu release version:\n%s", updatedYaml) + } + if !strings.Contains(updatedYaml, newTkgBundle.TkgVersion) || strings.Contains(updatedYaml, oldTkgBundle.TkgVersion) { + t.Fatalf("failed updating the Tanzu grid version:\n%s", updatedYaml) + } + if !strings.Contains(updatedYaml, newTkgBundle.CoreDnsVersion) || strings.Contains(updatedYaml, oldTkgBundle.CoreDnsVersion) { + t.Fatalf("failed updating the CoreDNS version:\n%s", updatedYaml) + } + if !strings.Contains(updatedYaml, newTkgBundle.EtcdVersion) || strings.Contains(updatedYaml, oldTkgBundle.EtcdVersion) { + t.Fatalf("failed updating the Etcd version:\n%s", updatedYaml) } } @@ -251,7 +275,7 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { } // Enables Machine Health Check - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, &machineHealthCheck{ + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, &cseMachineHealthCheckInternal{ MaxUnhealthyNodesPercentage: 12, NodeStartupTimeout: "34", NodeNotReadyTimeout: "56", diff --git a/types/v56/cse.go b/types/v56/cse.go index a3e49fc60..55c2df2e1 100644 --- a/types/v56/cse.go +++ b/types/v56/cse.go @@ -5,179 +5,169 @@ import "time" // Capvcd (Cluster API Provider for VCD), is a type that represents a Kubernetes cluster inside VCD, that is created and managed // with the Container Service Extension (CSE) type Capvcd struct { - Kind string `json:"kind,omitempty"` - Name string `json:"name,omitempty"` + Kind string `json:"kind"` Spec struct { VcdKe struct { - ForceDelete bool `json:"forceDelete,omitempty"` - MarkForDelete bool `json:"markForDelete,omitempty"` - IsVCDKECluster bool `json:"isVCDKECluster,omitempty"` - AutoRepairOnErrors bool `json:"autoRepairOnErrors,omitempty"` + IsVCDKECluster bool `json:"isVCDKECluster"` + AutoRepairOnErrors bool `json:"autoRepairOnErrors"` DefaultStorageClassOptions struct { - Filesystem string `json:"filesystem,omitempty"` - K8SStorageClassName string `json:"k8sStorageClassName,omitempty"` - VcdStorageProfileName string `json:"vcdStorageProfileName,omitempty"` - UseDeleteReclaimPolicy bool `json:"useDeleteReclaimPolicy,omitempty"` - } `json:"defaultStorageClassOptions,omitempty"` - } `json:"vcdKe,omitempty"` - CapiYaml string `json:"capiYaml,omitempty"` - } `json:"spec,omitempty"` + Filesystem string `json:"filesystem"` + K8SStorageClassName string `json:"k8sStorageClassName"` + VcdStorageProfileName string `json:"vcdStorageProfileName"` + UseDeleteReclaimPolicy bool `json:"useDeleteReclaimPolicy"` + } `json:"defaultStorageClassOptions"` + } `json:"vcdKe"` + CapiYaml string `json:"capiYaml"` + } `json:"spec"` Status struct { Cpi struct { - Name string `json:"name,omitempty"` - Version string `json:"version,omitempty"` + Name string `json:"name"` + Version string `json:"version"` EventSet []struct { - Name string `json:"name,omitempty"` - OccurredAt time.Time `json:"occurredAt,omitempty"` - VcdResourceId string `json:"vcdResourceId,omitempty"` + Name string `json:"name"` + OccurredAt time.Time `json:"occurredAt"` + VcdResourceId string `json:"vcdResourceId"` AdditionalDetails struct { - DetailedEvent string `json:"Detailed Event,omitempty"` - } `json:"additionalDetails,omitempty"` - } `json:"eventSet,omitempty"` - } `json:"cpi,omitempty"` + DetailedEvent string `json:"Detailed Event"` + } `json:"additionalDetails"` + } `json:"eventSet"` + } `json:"cpi"` Csi struct { - Name string `json:"name,omitempty"` - Version string `json:"version,omitempty"` + Name string `json:"name"` + Version string `json:"version"` EventSet []struct { - Name string `json:"name,omitempty"` - OccurredAt time.Time `json:"occurredAt,omitempty"` + Name string `json:"name"` + OccurredAt time.Time `json:"occurredAt"` AdditionalDetails struct { DetailedDescription string `json:"Detailed Description,omitempty"` - } `json:"additionalDetails,omitempty"` - } `json:"eventSet,omitempty"` - } `json:"csi,omitempty"` + } `json:"additionalDetails"` + } `json:"eventSet"` + } `json:"csi"` VcdKe struct { - State string `json:"state,omitempty"` + State string `json:"state"` EventSet []struct { - Name string `json:"name,omitempty"` - OccurredAt time.Time `json:"occurredAt,omitempty"` - VcdResourceId string `json:"vcdResourceId,omitempty"` + Name string `json:"name"` + OccurredAt time.Time `json:"occurredAt"` + VcdResourceId string `json:"vcdResourceId"` AdditionalDetails struct { - DetailedEvent string `json:"Detailed Event,omitempty"` - } `json:"additionalDetails,omitempty"` - } `json:"eventSet,omitempty"` - WorkerId string `json:"workerId,omitempty"` - VcdKeVersion string `json:"vcdKeVersion,omitempty"` + DetailedEvent string `json:"Detailed Event"` + } `json:"additionalDetails"` + } `json:"eventSet"` + WorkerId string `json:"workerId"` + VcdKeVersion string `json:"vcdKeVersion"` VcdResourceSet []struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - } `json:"vcdResourceSet,omitempty"` - HeartbeatString string `json:"heartbeatString,omitempty"` - VcdKeInstanceId string `json:"vcdKeInstanceId,omitempty"` - HeartbeatTimestamp string `json:"heartbeatTimestamp,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + } `json:"vcdResourceSet"` + HeartbeatString string `json:"heartbeatString"` + VcdKeInstanceId string `json:"vcdKeInstanceId"` + HeartbeatTimestamp string `json:"heartbeatTimestamp"` DefaultStorageClass struct { - FileSystem string `json:"fileSystem,omitempty"` - K8SStorageClassName string `json:"k8sStorageClassName,omitempty"` - VcdStorageProfileName string `json:"vcdStorageProfileName,omitempty"` - UseDeleteReclaimPolicy bool `json:"useDeleteReclaimPolicy,omitempty"` - } `json:"defaultStorageClass,omitempty"` - } `json:"vcdKe,omitempty"` + FileSystem string `json:"fileSystem"` + K8SStorageClassName string `json:"k8sStorageClassName"` + VcdStorageProfileName string `json:"vcdStorageProfileName"` + UseDeleteReclaimPolicy bool `json:"useDeleteReclaimPolicy"` + } `json:"defaultStorageClass"` + } `json:"vcdKe"` Capvcd struct { - Uid string `json:"uid,omitempty"` - Phase string `json:"phase,omitempty"` + Uid string `json:"uid"` + Phase string `json:"phase"` Upgrade struct { - Ready bool `json:"ready,omitempty"` + Ready bool `json:"ready"` Current struct { - TkgVersion string `json:"tkgVersion,omitempty"` - KubernetesVersion string `json:"kubernetesVersion,omitempty"` - } `json:"current,omitempty"` - } `json:"upgrade,omitempty"` + TkgVersion string `json:"tkgVersion"` + KubernetesVersion string `json:"kubernetesVersion"` + } `json:"current"` + } `json:"upgrade"` EventSet []struct { - Name string `json:"name,omitempty"` - OccurredAt time.Time `json:"occurredAt,omitempty"` - VcdResourceId string `json:"vcdResourceId,omitempty"` - VcdResourceName string `json:"vcdResourceName,omitempty"` - AdditionalDetails struct { - Event string `json:"event,omitempty"` - } `json:"additionalDetails,omitempty"` - } `json:"eventSet,omitempty"` - ErrorSet []struct { - Name string `json:"name,omitempty"` - OccurredAt time.Time `json:"occurredAt,omitempty"` - VcdResourceId string `json:"vcdResourceId,omitempty"` + Name string `json:"name"` + OccurredAt time.Time `json:"occurredAt"` + VcdResourceId string `json:"vcdResourceId"` VcdResourceName string `json:"vcdResourceName,omitempty"` AdditionalDetails struct { - DetailedError string `json:"Detailed Error,omitempty"` + Event string `json:"event"` } `json:"additionalDetails,omitempty"` - } `json:"errorSet,omitempty"` + } `json:"eventSet"` NodePool []struct { - Name string `json:"name,omitempty"` - DiskSizeMb int `json:"diskSizeMb,omitempty"` - NodeStatus map[string]interface{} `json:"nodeStatus,omitempty"` - SizingPolicy string `json:"sizingPolicy,omitempty"` - PlacementPolicy string `json:"placementPolicy,omitempty"` - NvidiaGpuEnabled bool `json:"nvidiaGpuEnabled,omitempty"` - StorageProfile string `json:"storageProfile,omitempty"` - DesiredReplicas int `json:"desiredReplicas,omitempty"` - AvailableReplicas int `json:"availableReplicas,omitempty"` - } `json:"nodePool,omitempty"` - ParentUid string `json:"parentUid,omitempty"` - K8sNetwork struct { + Name string `json:"name"` + DiskSizeMb int `json:"diskSizeMb"` + NodeStatus struct { + CseTest1WorkerNodePool1774Bdcdbffxcwc4BG9Nh9 string `json:"cse-test1-worker-node-pool-1-774bdcdbffxcwc4b-g9nh9,omitempty"` + CseTest1WorkerNodePool1774Bdcdbffxcwc4BRx9Wf string `json:"cse-test1-worker-node-pool-1-774bdcdbffxcwc4b-rx9wf,omitempty"` + CseTest1ControlPlaneNodePool56Jhv string `json:"cse-test1-control-plane-node-pool-56jhv,omitempty"` + } `json:"nodeStatus"` + SizingPolicy string `json:"sizingPolicy"` + StorageProfile string `json:"storageProfile"` + DesiredReplicas int `json:"desiredReplicas"` + AvailableReplicas int `json:"availableReplicas"` + } `json:"nodePool"` + ParentUid string `json:"parentUid"` + K8SNetwork struct { Pods struct { - CidrBlocks []string `json:"cidrBlocks,omitempty"` - } `json:"pods,omitempty"` + CidrBlocks []string `json:"cidrBlocks"` + } `json:"pods"` Services struct { - CidrBlocks []string `json:"cidrBlocks,omitempty"` - } `json:"services,omitempty"` - } `json:"k8sNetwork,omitempty"` - Kubernetes string `json:"kubernetes,omitempty"` - CapvcdVersion string `json:"capvcdVersion,omitempty"` + CidrBlocks []string `json:"cidrBlocks"` + } `json:"services"` + } `json:"k8sNetwork"` + Kubernetes string `json:"kubernetes"` + CapvcdVersion string `json:"capvcdVersion"` VcdProperties struct { - Site string `json:"site,omitempty"` + Site string `json:"site"` OrgVdcs []struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - OvdcNetworkName string `json:"ovdcNetworkName,omitempty"` - } `json:"orgVdcs,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + OvdcNetworkName string `json:"ovdcNetworkName"` + } `json:"orgVdcs"` Organizations []struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - } `json:"organizations,omitempty"` - } `json:"vcdProperties,omitempty"` - CapiStatusYaml string `json:"capiStatusYaml,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + } `json:"organizations"` + } `json:"vcdProperties"` + CapiStatusYaml string `json:"capiStatusYaml"` VcdResourceSet []struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` + Id string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` AdditionalDetails struct { - VirtualIP string `json:"virtualIP,omitempty"` + VirtualIP string `json:"virtualIP"` } `json:"additionalDetails,omitempty"` - } `json:"vcdResourceSet,omitempty"` + } `json:"vcdResourceSet"` ClusterApiStatus struct { - Phase string `json:"phase,omitempty"` + Phase string `json:"phase"` ApiEndpoints []struct { - Host string `json:"host,omitempty"` - Port int `json:"port,omitempty"` - } `json:"apiEndpoints,omitempty"` - } `json:"clusterApiStatus,omitempty"` - CreatedByVersion string `json:"createdByVersion,omitempty"` + Host string `json:"host"` + Port int `json:"port"` + } `json:"apiEndpoints"` + } `json:"clusterApiStatus"` + CreatedByVersion string `json:"createdByVersion"` ClusterResourceSetBindings []struct { - Kind string `json:"kind,omitempty"` - Name string `json:"name,omitempty"` - Applied bool `json:"applied,omitempty"` - LastAppliedTime string `json:"lastAppliedTime,omitempty"` - ClusterResourceSetName string `json:"clusterResourceSetName,omitempty"` - } `json:"clusterResourceSetBindings,omitempty"` - } `json:"capvcd,omitempty"` + Kind string `json:"kind"` + Name string `json:"name"` + Applied bool `json:"applied"` + LastAppliedTime string `json:"lastAppliedTime"` + ClusterResourceSetName string `json:"clusterResourceSetName"` + } `json:"clusterResourceSetBindings"` + } `json:"capvcd"` Projector struct { - Name string `json:"name,omitempty"` - Version string `json:"version,omitempty"` + Name string `json:"name"` + Version string `json:"version"` EventSet []struct { - Name string `json:"name,omitempty"` - OccurredAt time.Time `json:"occurredAt,omitempty"` - VcdResourceName string `json:"vcdResourceName,omitempty"` + Name string `json:"name"` + OccurredAt time.Time `json:"occurredAt"` + VcdResourceName string `json:"vcdResourceName"` AdditionalDetails struct { - Event string `json:"event,omitempty"` - } `json:"additionalDetails,omitempty"` - } `json:"eventSet,omitempty"` - } `json:"projector,omitempty"` - } `json:"status,omitempty"` + Event string `json:"event"` + } `json:"additionalDetails"` + } `json:"eventSet"` + } `json:"projector"` + } `json:"status"` Metadata struct { - Name string `json:"name,omitempty"` - Site string `json:"site,omitempty"` - OrgName string `json:"orgName,omitempty"` - VirtualDataCenterName string `json:"virtualDataCenterName,omitempty"` - } `json:"metadata,omitempty"` - ApiVersion string `json:"apiVersion,omitempty"` + Name string `json:"name"` + Site string `json:"site"` + OrgName string `json:"orgName"` + VirtualDataCenterName string `json:"virtualDataCenterName"` + } `json:"metadata"` + ApiVersion string `json:"apiVersion"` } From 2c5014d5989bb504f93dcff9223b3881fa5d150c Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 2 Feb 2024 14:01:52 +0100 Subject: [PATCH 016/115] Refactoring and trying to complete Reads Signed-off-by: abarreiro --- govcd/access_control.go | 2 +- govcd/cse.go | 47 ++++- govcd/cse_template.go | 23 +-- govcd/cse_test.go | 38 +++- govcd/cse_type.go | 34 ++-- govcd/cse_util.go | 367 +++++++++++++++++++++--------------- govcd/cse_yaml.go | 12 +- govcd/cse_yaml_unit_test.go | 2 +- types/v56/cse.go | 6 + 9 files changed, 328 insertions(+), 203 deletions(-) diff --git a/govcd/access_control.go b/govcd/access_control.go index 140722e4f..edeea4a47 100644 --- a/govcd/access_control.go +++ b/govcd/access_control.go @@ -15,7 +15,7 @@ import ( "github.com/vmware/go-vcloud-director/v2/types/v56" ) -// orgInfoCache is a cache to save org information, avoid repeated calls to compute the same result. +// orgInfoCache is a nameToIdCache to save org information, avoid repeated calls to compute the same result. // The keys to this map are the requesting objects IDs. var orgInfoCache = make(map[string]*TenantContext) diff --git a/govcd/cse.go b/govcd/cse.go index ed3d545ba..679ad8e60 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -9,6 +9,16 @@ import ( "time" ) +// supportedCseVersions is a map that contains only the supported CSE versions with the versions of its subcomponents. +// TODO: Is this really necessary? What happens in UI if I have a 1.1.0-1.2.0-1.0.0 (4.2) cluster and then CSE is updated to 4.3? +var supportedCseVersions = cseVersions{ + "4.2": { + VcdKeConfigRdeTypeVersion: "1.1.0", + CapvcdRdeTypeVersion: "1.2.0", + CseInterfaceVersion: "1.0.0", + }, +} + // CseCreateKubernetesCluster creates a Kubernetes cluster with the data given as input (CseClusterSettings). If the given // timeout is 0, it waits forever for the cluster creation. Otherwise, if the timeout is reached and the cluster is not available // (in "provisioned" state), it will return an error (the cluster will be left in VCD in any state) and the latest status @@ -32,7 +42,11 @@ func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterSettings, timeo // wait for the creation process to finish, so it doesn't monitor for any errors during the process. It returns just the ID of // the created cluster. One can manually check the status of the cluster with Org.CseGetKubernetesClusterById and the result of this method. func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) (string, error) { - goTemplateContents, err := cseClusterSettingsToInternal(clusterData, org) + if org == nil { + return "", fmt.Errorf("receiver Organization is nil") + } + + goTemplateContents, err := cseClusterSettingsToInternal(clusterData, *org) if err != nil { return "", err } @@ -42,7 +56,7 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) return "", err } - rde, err := createRdeAndPoll(org.client, "vmware", "capvcdCluster", supportedCseVersions[clusterData.CseVersion][1], types.DefinedEntity{ + rde, err := createRdeAndPoll(org.client, "vmware", "capvcdCluster", supportedCseVersions[clusterData.CseVersion].CapvcdRdeTypeVersion, types.DefinedEntity{ EntityType: goTemplateContents.RdeType.ID, Name: goTemplateContents.Name, Entity: rdeContents, @@ -67,7 +81,24 @@ func (org *Org) CseGetKubernetesClusterById(id string) (*CseKubernetesCluster, e if rde.DefinedEntity.Org.ID != org.Org.ID { return nil, fmt.Errorf("could not find any Kubernetes cluster with ID '%s' in Organization '%s': %s", id, org.Org.Name, ErrorEntityNotFound) } - return cseConvertToCseClusterApiProviderClusterType(rde) + return cseConvertToCseKubernetesClusterType(rde, nil) +} + +// GetKubeconfig retrieves the Kubeconfig from an available cluster. +func (cluster *CseKubernetesCluster) GetKubeconfig() (string, error) { + rde, err := getRdeById(cluster.client, cluster.ID) + if err != nil { + return "", err + } + var capvcd types.Capvcd + err = rde.InvokeBehaviorAndMarshal("", types.BehaviorInvocation{}, &capvcd) + if err != nil { + return "", err + } + if capvcd.Status.Capvcd.Private.KubeConfig == "" { + return "", fmt.Errorf("could not retrieve the Kubeconfig from the invocation of the Behavior") + } + return capvcd.Status.Capvcd.Private.KubeConfig, nil } // Refresh gets the latest information about the receiver cluster and updates its properties. @@ -76,7 +107,8 @@ func (cluster *CseKubernetesCluster) Refresh() error { if err != nil { return err } - refreshed, err := cseConvertToCseClusterApiProviderClusterType(rde) + cluster.nameToIdCache = nil // We deliberately want to refresh everything + refreshed, err := cseConvertToCseKubernetesClusterType(rde, cluster.nameToIdCache) if err != nil { return err } @@ -253,11 +285,10 @@ func (cluster *CseKubernetesCluster) Delete(timeoutMinutes time.Duration) error rde.DefinedEntity.Entity["spec"].(map[string]interface{})["vcdKe"] = vcdKe err = rde.Update(*rde.DefinedEntity) if err != nil { - if strings.Contains(strings.ToLower(err.Error()), "etag") { - continue // We ignore any ETag error. This just means a clash with the CSE Server, we just try again - // FIXME: No sleep here + // We ignore any ETag error. This just means a clash with the CSE Server, we just try again + if !strings.Contains(strings.ToLower(err.Error()), "etag") { + return fmt.Errorf("could not mark the Kubernetes cluster with ID '%s' to be deleted: %s", cluster.ID, err) } - return fmt.Errorf("could not mark the Kubernetes cluster with ID '%s' to be deleted: %s", cluster.ID, err) } } diff --git a/govcd/cse_template.go b/govcd/cse_template.go index c58910445..fbf144deb 100644 --- a/govcd/cse_template.go +++ b/govcd/cse_template.go @@ -14,7 +14,7 @@ import ( // getCseKubernetesClusterCreationPayload gets the payload for the RDE that will trigger a Kubernetes cluster creation. // It generates a valid YAML that is embedded inside the RDE JSON, then it is returned as an unmarshaled // generic map, that allows to be sent to VCD as it is. -func getCseKubernetesClusterCreationPayload(goTemplateContents *cseClusterSettingsInternal) (map[string]interface{}, error) { +func getCseKubernetesClusterCreationPayload(goTemplateContents cseClusterSettingsInternal) (map[string]interface{}, error) { capiYaml, err := generateCapiYaml(goTemplateContents) if err != nil { return nil, err @@ -60,7 +60,7 @@ func getCseKubernetesClusterCreationPayload(goTemplateContents *cseClusterSettin } // generateNodePoolYaml generates YAML blocks corresponding to the Kubernetes node pools. -func generateNodePoolYaml(clusterDetails *cseClusterSettingsInternal) (string, error) { +func generateNodePoolYaml(clusterDetails cseClusterSettingsInternal) (string, error) { workerPoolTmpl, err := getCseTemplate(clusterDetails.CseVersion, "capiyaml_workerpool") if err != nil { return "", err @@ -105,8 +105,9 @@ func generateNodePoolYaml(clusterDetails *cseClusterSettingsInternal) (string, e } // generateMemoryHealthCheckYaml generates a YAML block corresponding to the Kubernetes memory health check. -func generateMemoryHealthCheckYaml(mhcSettings *cseMachineHealthCheckInternal, cseVersion, clusterName string) (string, error) { - if mhcSettings == nil { +func generateMemoryHealthCheckYaml(vcdKeConfig vcdKeConfig, cseVersion, clusterName string) (string, error) { + if vcdKeConfig.NodeStartupTimeout == "" && vcdKeConfig.NodeUnknownTimeout == "" && vcdKeConfig.NodeNotReadyTimeout == "" && + vcdKeConfig.MaxUnhealthyNodesPercentage == 0 { return "", nil } @@ -121,10 +122,10 @@ func generateMemoryHealthCheckYaml(mhcSettings *cseMachineHealthCheckInternal, c if err := mhcEmptyTmpl.Execute(buf, map[string]string{ "ClusterName": clusterName, "TargetNamespace": clusterName + "-ns", - "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", mhcSettings.MaxUnhealthyNodesPercentage), // With the 'percentage' suffix - "NodeStartupTimeout": fmt.Sprintf("%ss", mhcSettings.NodeStartupTimeout), // With the 'second' suffix - "NodeUnknownTimeout": fmt.Sprintf("%ss", mhcSettings.NodeUnknownTimeout), // With the 'second' suffix - "NodeNotReadyTimeout": fmt.Sprintf("%ss", mhcSettings.NodeNotReadyTimeout), // With the 'second' suffix + "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", vcdKeConfig.MaxUnhealthyNodesPercentage), // With the 'percentage' suffix + "NodeStartupTimeout": fmt.Sprintf("%ss", vcdKeConfig.NodeStartupTimeout), // With the 'second' suffix + "NodeUnknownTimeout": fmt.Sprintf("%ss", vcdKeConfig.NodeUnknownTimeout), // With the 'second' suffix + "NodeNotReadyTimeout": fmt.Sprintf("%ss", vcdKeConfig.NodeNotReadyTimeout), // With the 'second' suffix }); err != nil { return "", fmt.Errorf("could not generate a correct Memory Health Check YAML: %s", err) } @@ -135,7 +136,7 @@ func generateMemoryHealthCheckYaml(mhcSettings *cseMachineHealthCheckInternal, c // generateCapiYaml generates the YAML string that is required during Kubernetes cluster creation, to be embedded // in the CAPVCD cluster JSON payload. This function picks data from the Terraform schema and the createClusterDto to // populate several Go templates and build a final YAML. -func generateCapiYaml(clusterDetails *cseClusterSettingsInternal) (string, error) { +func generateCapiYaml(clusterDetails cseClusterSettingsInternal) (string, error) { clusterTmpl, err := getCseTemplate(clusterDetails.CseVersion, "capiyaml_cluster") if err != nil { return "", err @@ -150,7 +151,7 @@ func generateCapiYaml(clusterDetails *cseClusterSettingsInternal) (string, error return "", err } - memoryHealthCheckYaml, err := generateMemoryHealthCheckYaml(clusterDetails.MachineHealthCheck, clusterDetails.CseVersion, clusterDetails.Name) + memoryHealthCheckYaml, err := generateMemoryHealthCheckYaml(clusterDetails.VcdKeConfig, clusterDetails.CseVersion, clusterDetails.Name) if err != nil { return "", err } @@ -178,7 +179,7 @@ func generateCapiYaml(clusterDetails *cseClusterSettingsInternal) (string, error "ControlPlaneEndpoint": clusterDetails.ControlPlane.Ip, "DnsVersion": clusterDetails.TkgVersionBundle.CoreDnsVersion, "EtcdVersion": clusterDetails.TkgVersionBundle.EtcdVersion, - "ContainerRegistryUrl": clusterDetails.ContainerRegistryUrl, + "ContainerRegistryUrl": clusterDetails.VcdKeConfig.ContainerRegistryUrl, "KubernetesVersion": clusterDetails.TkgVersionBundle.KubernetesVersion, "SshPublicKey": clusterDetails.SshPublicKey, "VirtualIpSubnet": clusterDetails.VirtualIpSubnet, diff --git a/govcd/cse_test.go b/govcd/cse_test.go index a4b96ad1f..5bdba8214 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -12,6 +12,7 @@ import ( . "gopkg.in/check.v1" "net/url" "os" + "strings" ) const ( @@ -41,6 +42,8 @@ func (vcd *TestVCD) Test_Cse(check *C) { ova, err := catalog.GetVAppTemplateByName(vcd.config.Cse.OvaName) check.Assert(err, IsNil) + tkgBundle, err := getTkgVersionBundleFromVAppTemplateName(ova.VAppTemplate.Name) + check.Assert(err, IsNil) vdc, err := org.GetVDCByName(vcd.config.Cse.TenantVdc, false) check.Assert(err, IsNil) @@ -64,9 +67,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { apiToken, err := token.GetInitialApiToken() check.Assert(err, IsNil) - workerPoolName := "worker-pool-1" - - cluster, err := org.CseCreateKubernetesCluster(CseClusterSettings{ + clusterSettings := CseClusterSettings{ Name: "test-cse", OrganizationId: org.Org.ID, VdcId: vdc.Vdc.ID, @@ -81,7 +82,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { Ip: "", }, WorkerPools: []CseWorkerPoolSettings{{ - Name: workerPoolName, + Name: "worker-pool-1", MachineCount: 1, DiskSizeGi: 20, SizingPolicyId: policies[0].VdcComputePolicyV2.ID, @@ -99,11 +100,28 @@ func (vcd *TestVCD) Test_Cse(check *C) { PodCidr: "100.96.0.0/11", ServiceCidr: "100.64.0.0/13", AutoRepairOnErrors: true, - }, 0) + } + + cluster, err := org.CseCreateKubernetesCluster(clusterSettings, 0) check.Assert(err, IsNil) - check.Assert(cluster.ID, Not(Equals), "") + check.Assert(true, Equals, strings.Contains(cluster.ID, "urn:vcloud:entity:vmware:capvcdCluster:")) check.Assert(cluster.Etag, Not(Equals), "") - check.Assert(cluster.capvcdType.Status.VcdKe.State, Equals, "provisioned") + check.Assert(cluster.CseClusterSettings, DeepEquals, clusterSettings) + check.Assert(cluster.KubernetesVersion, Equals, tkgBundle.KubernetesVersion) + check.Assert(cluster.TkgVersion, Equals, tkgBundle.TkgVersion) + check.Assert(cluster.CapvcdVersion, Not(Equals), "") + check.Assert(cluster.CpiVersion, Not(Equals), "") + check.Assert(cluster.CsiVersion, Not(Equals), "") + check.Assert(len(cluster.ClusterResourceSetBindings), Not(Equals), 0) + check.Assert(cluster.State, Equals, "provisioned") + check.Assert(len(cluster.Events), Not(Equals), 0) + + kubeconfig, err := cluster.GetKubeconfig() + check.Assert(err, IsNil) + check.Assert(true, Equals, strings.Contains(kubeconfig, cluster.Name)) + check.Assert(true, Equals, strings.Contains(kubeconfig, "client-certificate-data")) + check.Assert(true, Equals, strings.Contains(kubeconfig, "certificate-authority-data")) + check.Assert(true, Equals, strings.Contains(kubeconfig, "client-key-data")) err = cluster.Refresh() check.Assert(err, IsNil) @@ -119,18 +137,18 @@ func (vcd *TestVCD) Test_Cse(check *C) { // Update worker pool from 1 node to 2 // Pre-check. This should be 1, as it was created with just 1 pool for _, nodePool := range cluster.capvcdType.Status.Capvcd.NodePool { - if nodePool.Name == workerPoolName { + if nodePool.Name == clusterSettings.WorkerPools[0].Name { check.Assert(nodePool.DesiredReplicas, Equals, 1) } } // Perform the update - err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}) + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{clusterSettings.WorkerPools[0].Name: {MachineCount: 2}}) check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up foundWorkerPool := false for _, nodePool := range cluster.capvcdType.Status.Capvcd.NodePool { - if nodePool.Name == workerPoolName { + if nodePool.Name == clusterSettings.WorkerPools[0].Name { foundWorkerPool = true check.Assert(nodePool.DesiredReplicas, Equals, 2) } diff --git a/govcd/cse_type.go b/govcd/cse_type.go index be795e22f..531bdb535 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -18,16 +18,18 @@ type CseKubernetesCluster struct { CpiVersion string CsiVersion string State string - Kubeconfig string Events []CseClusterEvent client *Client capvcdType *types.Capvcd + + nameToIdCache map[string]string // This helps to reduce calls to VCD drastically, specially when doing item Name->ID transformations inside loops } // CseClusterEvent is an event that has occurred during the lifetime of a Container Service Extension (CSE) Kubernetes cluster. type CseClusterEvent struct { Name string + Type string OccurredAt time.Time Details string } @@ -128,11 +130,10 @@ type cseClusterSettingsInternal struct { ControlPlane cseControlPlaneSettingsInternal WorkerPools []cseWorkerPoolSettingsInternal DefaultStorageClass cseDefaultStorageClassInternal - MachineHealthCheck *cseMachineHealthCheckInternal + VcdKeConfig vcdKeConfig Owner string ApiToken string VcdUrl string - ContainerRegistryUrl string VirtualIpSubnet string SshPublicKey string PodCidr string @@ -169,29 +170,28 @@ type cseDefaultStorageClassInternal struct { Filesystem string } -// cseMachineHealthCheckInternal is a type that contains only the required and relevant fields from the VCDKEConfig (CSE Server) configuration, +// vcdKeConfig is a type that contains only the required and relevant fields from the VCDKEConfig (CSE Server) configuration, // such as the Machine Health Check settings. -type cseMachineHealthCheckInternal struct { +type vcdKeConfig struct { MaxUnhealthyNodesPercentage float64 NodeStartupTimeout string NodeNotReadyTimeout string NodeUnknownTimeout string + ContainerRegistryUrl string +} + +// cseComponentVersions is a type that registers the versions of the subcomponents of a specific CSE Version +type cseComponentVersions struct { + VcdKeConfigRdeTypeVersion string + CapvcdRdeTypeVersion string + CseInterfaceVersion string } +// cseVersions is a map that links a CSE Version with the versions of its subcomponents +type cseVersions map[string]cseComponentVersions + // This collection of files contains all the Go Templates and resources required for the Container Service Extension (CSE) methods // to work. // //go:embed cse var cseFiles embed.FS - -// supportedCseVersions is a map that contains only the supported CSE versions as keys, -// and its corresponding components versions as a slice of strings. The first string is the VCDKEConfig RDE Type version, -// then the CAPVCD RDE Type version and finally the CAPVCD Behavior version. -// TODO: Is this really necessary? What happens in UI if I have a 1.1.0-1.2.0-1.0.0 (4.2) cluster and then CSE is updated to 4.3? -var supportedCseVersions = map[string][]string{ - "4.2": { - "1.1.0", // VCDKEConfig RDE Type version - "1.2.0", // CAPVCD RDE Type version - "1.0.0", // CAPVCD Behavior version - }, -} diff --git a/govcd/cse_util.go b/govcd/cse_util.go index c5f512df1..b966adb1b 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -6,16 +6,18 @@ import ( "fmt" "github.com/vmware/go-vcloud-director/v2/types/v56" "github.com/vmware/go-vcloud-director/v2/util" + "net/url" "regexp" "strings" "time" ) -// cseConvertToCseClusterApiProviderClusterType takes a generic RDE that must represent an existing CSE Kubernetes cluster, -// and transforms it to a specific Container Service Extension CseKubernetesCluster object that represents the same cluster, but -// it is easy to explore and consume. If the receiver object does not contain a CAPVCD object, this method +// cseConvertToCseKubernetesClusterType takes a generic RDE that must represent an existing CSE Kubernetes cluster, +// and transforms it to an equivalent CseKubernetesCluster object that represents the same cluster, but +// it is easy to explore and consume. If the input RDE is not a CSE Kubernetes cluster, this method // will obviously return an error. -func cseConvertToCseClusterApiProviderClusterType(rde *DefinedEntity) (*CseKubernetesCluster, error) { +// The nameToIdCache maps names with their IDs. This is used to reduce calls to VCD to retrieve this information. +func cseConvertToCseKubernetesClusterType(rde *DefinedEntity, nameToIdCache map[string]string) (*CseKubernetesCluster, error) { requiredType := "vmware:capvcdCluster" if !strings.Contains(rde.DefinedEntity.ID, requiredType) || !strings.Contains(rde.DefinedEntity.EntityType, requiredType) { @@ -50,7 +52,12 @@ func cseConvertToCseClusterApiProviderClusterType(rde *DefinedEntity) (*CseKuber State: capvcd.Status.VcdKe.State, client: rde.client, capvcdType: capvcd, + nameToIdCache: nameToIdCache, } + if result.nameToIdCache == nil { + result.nameToIdCache = map[string]string{} + } + for i, binding := range capvcd.Status.Capvcd.ClusterResourceSetBindings { result.ClusterResourceSetBindings[i] = binding.ClusterResourceSetName } @@ -60,23 +67,46 @@ func cseConvertToCseClusterApiProviderClusterType(rde *DefinedEntity) (*CseKuber } result.OrganizationId = result.capvcdType.Status.Capvcd.VcdProperties.Organizations[0].Id if len(result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs) == 0 { - return nil, fmt.Errorf("could not read Org VDC Network from Capvcd type") + return nil, fmt.Errorf("could not read VDCs from Capvcd type") } result.VdcId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Id - result.NetworkId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName // TODO: ID + + // To retrieve the Network ID, we check that it is not already cached. If it's not, we retrieve it with + // the VDC ID and name filters + if _, ok := result.nameToIdCache[result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName]; !ok { + params := url.Values{} + params.Add("filter", fmt.Sprintf("name==%s", result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName)) + params = queryParameterFilterAnd("ownerRef.id=="+result.VdcId, params) + + networks, err := getAllOpenApiOrgVdcNetworks(rde.client, params) + if err != nil { + return nil, fmt.Errorf("could not read Org VDC Network from Capvcd type: %s", err) + } + if len(networks) != 1 { + return nil, fmt.Errorf("expected one Org VDC Network from Capvcd type, but got %d", len(networks)) + } + result.nameToIdCache[result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName] = networks[0].OpenApiOrgVdcNetwork.ID + } + result.NetworkId = result.nameToIdCache[result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName] if rde.DefinedEntity.Owner == nil { return nil, fmt.Errorf("could not read Owner from RDE") } result.Owner = rde.DefinedEntity.Owner.Name - if result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.K8SStorageClassName != "" { + if result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.K8SStorageClassName != "" { // This would mean there is a Default Storage Class defined result.DefaultStorageClass = &CseDefaultStorageClassSettings{ - StorageProfileId: "", // TODO: ID - Name: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.K8SStorageClassName, - ReclaimPolicy: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName, - Filesystem: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName, + Name: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.K8SStorageClassName, + ReclaimPolicy: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName, + Filesystem: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName, + } + + // To retrieve the Storage Profile ID, we check that it is not already cached. If it's not, we retrieve it + if _, ok := result.nameToIdCache[result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName]; !ok { + // TODO: There is no method to retrieve Storage profiles by name.... + result.nameToIdCache[result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName] = "" } + result.DefaultStorageClass.StorageProfileId = result.nameToIdCache[result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName] } yamlDocuments, err := unmarshalMultipleYamlDocuments(result.capvcdType.Spec.CapiYaml) @@ -84,115 +114,178 @@ func cseConvertToCseClusterApiProviderClusterType(rde *DefinedEntity) (*CseKuber return nil, err } - result.KubernetesTemplateOvaId = "" // TODO: YAML > ID - result.CseVersion = "" // TODO: Get opposite from supportedVersionsMap - // TODO: YAML > Control Plane - // TODO: YAML > Worker pools - // TODO: YAML > Health check - // YAML PodCidr: "", - // YAML ServiceCidr: "", - // YAML SshPublicKey: "", - // YAML VirtualIpSubnet: "", + result.CseVersion = "" // TODO: Get opposite from supportedVersionsMap + //var workerPools []CseWorkerPoolSettings for _, yamlDocument := range yamlDocuments { switch yamlDocument["kind"] { - + case "KubeadmControlPlane": + // result.ControlPlane.MachineCount + // result.SshPublicKey + case "VCDMachineTemplate": + // Obtain all name->ID + if strings.Contains("name", "control-plane-node-pool") { + // TODO: There is no method to retrieve vApp templates by name.... + // result.KubernetesTemplateOvaId + // TODO: There is no method to retrieve vApp templates by name.... + // getAllVdcComputePoliciesV2() + // result.ControlPlane.SizingPolicyId + // result.ControlPlane.PlacementPolicyId + // result.ControlPlane.StorageProfileId + // result.ControlPlane.DiskSizeGi + fmt.Print("b") + } else { + fmt.Print("a") + //workerPool := CseWorkerPoolSettings{} + //workerPools = append(workerPools, workerPool) + } + case "VCDCluster": + // result.ControlPlane.Ip + // result.VirtualIpSubnet + case "Cluster": + // result.PodCidr + // result.ServicesCidr + case "MachineHealthCheck": + // This is quite simple, if we find this document, means that Machine Health Check is enabled + result.NodeHealthCheck = true } } - if err != nil { - return nil, fmt.Errorf("could not get the cluster state from the RDE contents: %s", err) - } + + // // TODO: This needs a refactoring + // if nodePool.PlacementPolicy != "" { + // policies, err := vcdClient.GetAllVdcComputePoliciesV2(url.Values{ + // "filter": []string{fmt.Sprintf("name==%s", nodePool.PlacementPolicy)}, + // }) + // if err != nil { + // return nil, err // TODO + // } + // nameToIds[nodePool.PlacementPolicy] = policies[0].VdcComputePolicyV2.ID + // } + // if nodePool.SizingPolicy != "" { + // policies, err := vcdClient.GetAllVdcComputePoliciesV2(url.Values{ + // "filter": []string{fmt.Sprintf("name==%s", nodePool.SizingPolicy)}, + // }) + // if err != nil { + // return nil, err // TODO + // } + // nameToIds[nodePool.SizingPolicy] = policies[0].VdcComputePolicyV2.ID + // } + // if nodePool.StorageProfile != "" { + // ref, err := vdc.FindStorageProfileReference(nodePool.StorageProfile) + // if err != nil { + // return nil, fmt.Errorf("could not get Default Storage Class options from 'spec.vcdKe.defaultStorageClassOptions': %s", err) // TODO + // } + // nameToIds[nodePool.StorageProfile] = ref.ID + // } + // block["sizing_policy_id"] = nameToIds[nodePool.SizingPolicy] + // if nodePool.NvidiaGpuEnabled { // TODO: Be sure this is a worker node pool and not control plane (doesnt have this attr) + // block["vgpu_policy_id"] = nameToIds[nodePool.PlacementPolicy] // It's a placement policy here + // } else { + // block["placement_policy_id"] = nameToIds[nodePool.PlacementPolicy] + // } + // block["storage_profile_id"] = nameToIds[nodePool.StorageProfile] + // block["disk_size_gi"] = nodePool.DiskSizeMb / 1024 + // + // if strings.HasSuffix(nodePool.Name, "-control-plane-node-pool") { + // // Control Plane + // if len(cluster.Capvcd.Status.Capvcd.ClusterApiStatus.ApiEndpoints) == 0 { + // return nil, fmt.Errorf("could not retrieve Cluster IP") + // } + // block["ip"] = cluster.Capvcd.Status.Capvcd.ClusterApiStatus.ApiEndpoints[0].Host + // controlPlaneBlocks[0] = block + // } else { + // // Worker node + // block["name"] = nodePool.Name + // + // nodePoolBlocks[i] = block + // } return result, nil } // waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if timeoutMinutes = 0) -// or until this timeout is reached. If the cluster is in "provisioned" state before the given timeout, it returns a CseKubernetesCluster object -// representing the Kubernetes cluster with all the latest information. +// or until the timeout is reached. If the cluster is in "provisioned" state before the given timeout, it returns a CseKubernetesCluster object +// representing the Kubernetes cluster with all its latest details. // If one of the states of the cluster at a given point is "error", this function also checks whether the cluster has the "Auto Repair on Errors" flag enabled, // so it keeps waiting if it's true. -// If timeout is reached before the cluster, it returns an error. +// If timeout is reached before the cluster is in "provisioned" state, it returns an error. func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinutes time.Duration) (*CseKubernetesCluster, error) { var elapsed time.Duration - logHttpResponse := util.LogHttpResponse - sleepTime := 30 - - // The following loop is constantly polling VCD to retrieve the RDE, which has a big JSON inside, so we avoid filling - // the log with these big payloads. We use defer to be sure that we restore the initial logging state. - defer func() { - util.LogHttpResponse = logHttpResponse - }() + sleepTime := 10 start := time.Now() - var capvcdCluster *CseKubernetesCluster + cluster := &CseKubernetesCluster{} for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies timeoutMinutes=0, we wait forever - util.LogHttpResponse = false rde, err := getRdeById(client, clusterId) - util.LogHttpResponse = logHttpResponse if err != nil { return nil, err } - capvcdCluster, err = cseConvertToCseClusterApiProviderClusterType(rde) + cluster, err = cseConvertToCseKubernetesClusterType(rde, cluster.nameToIdCache) if err != nil { return nil, err } - switch capvcdCluster.capvcdType.Status.VcdKe.State { + switch cluster.State { case "provisioned": - return capvcdCluster, nil + return cluster, nil case "error": // We just finish if auto-recovery is disabled, otherwise we just let CSE fixing things in background - if !capvcdCluster.capvcdType.Spec.VcdKe.AutoRepairOnErrors { - // Try to give feedback about what went wrong, which is located in a set of events in the RDE payload - return capvcdCluster, fmt.Errorf("got an error and 'auto repair on errors' is disabled, aborting") - // TODO return capvcdCluster, fmt.Errorf("got an error and 'auto repair on errors' is disabled, aborting. Errors: %s", capvcdCluster.capvcdType.Status.Capvcd.ErrorSet[len(capvcdCluster.capvcdType.Status.Capvcd.ErrorSet)-1].AdditionalDetails.DetailedError) + if !cluster.AutoRepairOnErrors { + // Give feedback about what went wrong + errors := "" + for _, event := range cluster.Events { + if event.Type == "error" { + errors += fmt.Sprintf("%s\n", event.Details) + } + } + return cluster, fmt.Errorf("got an error and 'auto repair on errors' is disabled, aborting. Errors:\n%s", errors) } } - util.Logger.Printf("[DEBUG] Cluster '%s' is in '%s' state, will check again in %d seconds", capvcdCluster.ID, capvcdCluster.capvcdType.Status.VcdKe.State, sleepTime) + util.Logger.Printf("[DEBUG] Cluster '%s' is in '%s' state, will check again in %d seconds", cluster.ID, cluster.State, sleepTime) elapsed = time.Since(start) time.Sleep(time.Duration(sleepTime) * time.Second) } - return capvcdCluster, fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, capvcdCluster.capvcdType.Status.VcdKe.State) + return cluster, fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, cluster.State) } // validate validates the CSE Kubernetes cluster creation input data. Returns an error if some of the fields is wrong. -func (ccd *CseClusterSettings) validate() error { +func (input *CseClusterSettings) validate() error { cseNamesRegex, err := regexp.Compile(`^[a-z](?:[a-z0-9-]{0,29}[a-z0-9])?$`) if err != nil { return fmt.Errorf("could not compile regular expression '%s'", err) } - if !cseNamesRegex.MatchString(ccd.Name) { - return fmt.Errorf("the cluster name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", ccd.Name) + if !cseNamesRegex.MatchString(input.Name) { + return fmt.Errorf("the cluster name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", input.Name) } - if ccd.OrganizationId == "" { + if input.OrganizationId == "" { return fmt.Errorf("the Organization ID is required") } - if ccd.VdcId == "" { + if input.VdcId == "" { return fmt.Errorf("the VDC ID is required") } - if ccd.KubernetesTemplateOvaId == "" { + if input.KubernetesTemplateOvaId == "" { return fmt.Errorf("the Kubernetes template OVA ID is required") } - if ccd.NetworkId == "" { + if input.NetworkId == "" { return fmt.Errorf("the Network ID is required") } - if _, ok := supportedCseVersions[ccd.CseVersion]; !ok { - return fmt.Errorf("the CSE version '%s' is not supported. Must be one of %v", ccd.CseVersion, getKeys(supportedCseVersions)) + if _, ok := supportedCseVersions[input.CseVersion]; !ok { + return fmt.Errorf("the CSE version '%s' is not supported. Must be one of %v", input.CseVersion, getKeys(supportedCseVersions)) } - if ccd.ControlPlane.MachineCount < 1 || ccd.ControlPlane.MachineCount%2 == 0 { - return fmt.Errorf("number of control plane nodes must be odd and higher than 0, but it was '%d'", ccd.ControlPlane.MachineCount) + if input.ControlPlane.MachineCount < 1 || input.ControlPlane.MachineCount%2 == 0 { + return fmt.Errorf("number of control plane nodes must be odd and higher than 0, but it was '%d'", input.ControlPlane.MachineCount) } - if ccd.ControlPlane.DiskSizeGi < 20 { - return fmt.Errorf("disk size for the Control Plane in Gibibytes (Gi) must be at least 20, but it was '%d'", ccd.ControlPlane.DiskSizeGi) + if input.ControlPlane.DiskSizeGi < 20 { + return fmt.Errorf("disk size for the Control Plane in Gibibytes (Gi) must be at least 20, but it was '%d'", input.ControlPlane.DiskSizeGi) } - if len(ccd.WorkerPools) == 0 { + if len(input.WorkerPools) == 0 { return fmt.Errorf("there must be at least one Worker pool") } - for _, workerPool := range ccd.WorkerPools { + for _, workerPool := range input.WorkerPools { if !cseNamesRegex.MatchString(workerPool.Name) { return fmt.Errorf("the Worker pool name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", workerPool.Name) } @@ -203,27 +296,27 @@ func (ccd *CseClusterSettings) validate() error { return fmt.Errorf("number of Worker pool '%s' nodes must higher than 0, but it was '%d'", workerPool.Name, workerPool.MachineCount) } } - if ccd.DefaultStorageClass != nil { - if !cseNamesRegex.MatchString(ccd.DefaultStorageClass.Name) { - return fmt.Errorf("the Default Storage Class name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", ccd.DefaultStorageClass.Name) + if input.DefaultStorageClass != nil { + if !cseNamesRegex.MatchString(input.DefaultStorageClass.Name) { + return fmt.Errorf("the Default Storage Class name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", input.DefaultStorageClass.Name) } - if ccd.DefaultStorageClass.StorageProfileId == "" { + if input.DefaultStorageClass.StorageProfileId == "" { return fmt.Errorf("the Storage Profile ID for the Default Storage Class is required") } - if ccd.DefaultStorageClass.ReclaimPolicy != "delete" && ccd.DefaultStorageClass.ReclaimPolicy != "retain" { - return fmt.Errorf("the reclaim policy for the Default Storage Class must be either 'delete' or 'retain', but it was '%s'", ccd.DefaultStorageClass.ReclaimPolicy) + if input.DefaultStorageClass.ReclaimPolicy != "delete" && input.DefaultStorageClass.ReclaimPolicy != "retain" { + return fmt.Errorf("the reclaim policy for the Default Storage Class must be either 'delete' or 'retain', but it was '%s'", input.DefaultStorageClass.ReclaimPolicy) } - if ccd.DefaultStorageClass.Filesystem != "ext4" && ccd.DefaultStorageClass.ReclaimPolicy != "xfs" { - return fmt.Errorf("the filesystem for the Default Storage Class must be either 'ext4' or 'xfs', but it was '%s'", ccd.DefaultStorageClass.Filesystem) + if input.DefaultStorageClass.Filesystem != "ext4" && input.DefaultStorageClass.ReclaimPolicy != "xfs" { + return fmt.Errorf("the filesystem for the Default Storage Class must be either 'ext4' or 'xfs', but it was '%s'", input.DefaultStorageClass.Filesystem) } } - if ccd.ApiToken == "" { + if input.ApiToken == "" { return fmt.Errorf("the API token is required") } - if ccd.PodCidr == "" { + if input.PodCidr == "" { return fmt.Errorf("the Pod CIDR is required") } - if ccd.ServiceCidr == "" { + if input.ServiceCidr == "" { return fmt.Errorf("the Service CIDR is required") } @@ -232,58 +325,61 @@ func (ccd *CseClusterSettings) validate() error { // cseClusterSettingsToInternal transforms user input data (CseClusterSettings) into the final payload that // will be used to render the Go templates that define a Kubernetes cluster creation payload (cseClusterSettingsInternal). -func cseClusterSettingsToInternal(input CseClusterSettings, org *Org) (*cseClusterSettingsInternal, error) { +func cseClusterSettingsToInternal(input CseClusterSettings, org Org) (cseClusterSettingsInternal, error) { + output := cseClusterSettingsInternal{} err := input.validate() if err != nil { - return nil, err + return output, err } - if org == nil || org.Org == nil { - return nil, fmt.Errorf("cannot manipulate the CSE Kubernetes cluster creation input, the Organization is nil") + if org.Org == nil { + return output, fmt.Errorf("the Organization is nil") } - output := &cseClusterSettingsInternal{} output.OrganizationName = org.Org.Name vdc, err := org.GetVDCById(input.VdcId, true) if err != nil { - return nil, fmt.Errorf("could not retrieve the VDC with ID '%s': %s", input.VdcId, err) + return output, fmt.Errorf("could not retrieve the VDC with ID '%s': %s", input.VdcId, err) } output.VdcName = vdc.Vdc.Name vAppTemplate, err := getVAppTemplateById(org.client, input.KubernetesTemplateOvaId) if err != nil { - return nil, fmt.Errorf("could not retrieve the Kubernetes OVA with ID '%s': %s", input.KubernetesTemplateOvaId, err) + return output, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", input.KubernetesTemplateOvaId, err) } output.KubernetesTemplateOvaName = vAppTemplate.VAppTemplate.Name tkgVersions, err := getTkgVersionBundleFromVAppTemplateName(vAppTemplate.VAppTemplate.Name) if err != nil { - return nil, err + return output, fmt.Errorf("could not retrieve the required information from the Kubernetes Template OVA: %s", err) } output.TkgVersionBundle = tkgVersions catalogName, err := vAppTemplate.GetCatalogName() if err != nil { - return nil, fmt.Errorf("could not retrieve the Catalog name of the OVA '%s': %s", input.KubernetesTemplateOvaId, err) + return output, fmt.Errorf("could not retrieve the Catalog name where the the Kubernetes Template OVA '%s' is hosted: %s", input.KubernetesTemplateOvaId, err) } output.CatalogName = catalogName network, err := vdc.GetOrgVdcNetworkById(input.NetworkId, true) if err != nil { - return nil, fmt.Errorf("could not retrieve the Org VDC Network with ID '%s': %s", input.NetworkId, err) + return output, fmt.Errorf("could not retrieve the Org VDC Network with ID '%s': %s", input.NetworkId, err) } output.NetworkName = network.OrgVDCNetwork.Name - currentCseVersion := supportedCseVersions[input.CseVersion] - rdeType, err := getRdeType(org.client, "vmware", "capvcdCluster", currentCseVersion[1]) + currentCseVersion, ok := supportedCseVersions[input.CseVersion] + if !ok { + return output, fmt.Errorf("the CSE version '%s' is not supported. List of supported versions: %v", input.CseVersion, getKeys(supportedCseVersions)) + } + rdeType, err := getRdeType(org.client, "vmware", "capvcdCluster", currentCseVersion.CapvcdRdeTypeVersion) if err != nil { - return nil, fmt.Errorf("could not retrieve RDE Type vmware:capvcdCluster:'%s': %s", currentCseVersion[1], err) + return output, err } output.RdeType = rdeType.DefinedEntityType // The input to create a cluster uses different entities IDs, but CSE cluster creation process uses names. - // For that reason, we need to transform IDs to Names by querying VCD. This process is optimized with a tiny "cache" map. + // For that reason, we need to transform IDs to Names by querying VCD. This process is optimized with a tiny "nameToIdCache" map. idToNameCache := map[string]string{ "": "", // Default empty value to map optional values that were not set } @@ -300,7 +396,7 @@ func cseClusterSettingsToInternal(input CseClusterSettings, org *Org) (*cseClust if _, alreadyPresent := idToNameCache[id]; !alreadyPresent { storageProfile, err := getStorageProfileById(org.client, id) if err != nil { - return nil, fmt.Errorf("could not get Storage Profile with ID '%s': %s", id, err) + return output, fmt.Errorf("could not retrieve Storage Profile with ID '%s': %s", id, err) } idToNameCache[id] = storageProfile.Name } @@ -309,7 +405,7 @@ func cseClusterSettingsToInternal(input CseClusterSettings, org *Org) (*cseClust if _, alreadyPresent := idToNameCache[id]; !alreadyPresent { computePolicy, err := getVdcComputePolicyV2ById(org.client, id) if err != nil { - return nil, fmt.Errorf("could not get Compute Policy with ID '%s': %s", id, err) + return output, fmt.Errorf("could not retrieve Compute Policy with ID '%s': %s", id, err) } idToNameCache[id] = computePolicy.VdcComputePolicyV2.Name } @@ -349,26 +445,17 @@ func cseClusterSettingsToInternal(input CseClusterSettings, org *Org) (*cseClust } } - mhc, err := getMachineHealthCheck(org.client, supportedCseVersions[input.CseVersion][0], input.NodeHealthCheck) + vcdKeConfig, err := getVcdKeConfig(org.client, supportedCseVersions[input.CseVersion].VcdKeConfigRdeTypeVersion, input.NodeHealthCheck) if err != nil { - return nil, err - } - if mhc != nil { - output.MachineHealthCheck = mhc + return output, err } - - containerRegistryUrl, err := getContainerRegistryUrl(org.client, input.CseVersion) - if err != nil { - return nil, err - } - - output.ContainerRegistryUrl = containerRegistryUrl + output.VcdKeConfig = vcdKeConfig output.Owner = input.Owner if input.Owner == "" { sessionInfo, err := org.client.GetSessionInfo() if err != nil { - return nil, fmt.Errorf("error getting the owner of the cluster: %s", err) + return output, fmt.Errorf("error getting the owner of the cluster: %s", err) } output.Owner = sessionInfo.User.Name } @@ -423,77 +510,59 @@ func getTkgVersionBundleFromVAppTemplateName(ovaName string) (tkgVersionBundle, versionsMap := map[string]any{} err = json.Unmarshal(cseTkgVersionsJson, &versionsMap) if err != nil { - return result, err + return result, fmt.Errorf("failed unmarshaling cse/tkg_versions.json: %s", err) } versionMap, ok := versionsMap[parsedOvaName] if !ok { return result, fmt.Errorf("the Kubernetes OVA '%s' is not supported", parsedOvaName) } - // The map checking above guarantees that all splits and replaces will work - result.KubernetesVersion = strings.Split(parsedOvaName, "-")[0] - result.TkrVersion = strings.ReplaceAll(strings.Split(parsedOvaName, "-")[0], "+", "---") + "-" + strings.Split(parsedOvaName, "-")[1] + ovaParts := strings.Split(parsedOvaName, "-") + if len(ovaParts) < 2 { + return result, fmt.Errorf("unexpected error parsing the OVA name '%s', it doesn't follow the original naming convention", parsedOvaName) + } + + result.KubernetesVersion = ovaParts[0] + result.TkrVersion = strings.ReplaceAll(ovaParts[0], "+", "---") + "-" + ovaParts[1] result.TkgVersion = versionMap.(map[string]any)["tkg"].(string) result.EtcdVersion = versionMap.(map[string]any)["etcd"].(string) result.CoreDnsVersion = versionMap.(map[string]any)["coreDns"].(string) return result, nil } -// getMachineHealthCheck gets the required information from the CSE Server configuration RDE -func getMachineHealthCheck(client *Client, vcdKeConfigVersion string, isNodeHealthCheckActive bool) (*cseMachineHealthCheckInternal, error) { - if !isNodeHealthCheckActive { - return nil, nil - } - +// getVcdKeConfig gets the required information from the CSE Server configuration RDE +func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheckActive bool) (vcdKeConfig, error) { + result := vcdKeConfig{} rdes, err := getRdesByName(client, "vmware", "VCDKEConfig", vcdKeConfigVersion, "vcdKeConfig") if err != nil { - return nil, fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", vcdKeConfigVersion, err) + return result, fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", vcdKeConfigVersion, err) } if len(rdes) != 1 { - return nil, fmt.Errorf("expected exactly one VCDKEConfig RDE but got %d", len(rdes)) - } - // TODO: Get the struct Type for this one - profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]any) - if !ok { - return nil, fmt.Errorf("wrong format of VCDKEConfig, expected a 'profiles' array") - } - if len(profiles) != 1 { - return nil, fmt.Errorf("wrong format of VCDKEConfig, expected a single 'profiles' element, got %d", len(profiles)) + return result, fmt.Errorf("expected exactly one VCDKEConfig RDE but got %d", len(rdes)) } - mhc, ok := profiles[0].(map[string]any)["K8Config"].(map[string]any)["mhc"].(map[string]any) - if !ok { - return nil, nil - } - result := cseMachineHealthCheckInternal{} - result.MaxUnhealthyNodesPercentage = mhc["maxUnhealthyNodes"].(float64) - result.NodeStartupTimeout = mhc["nodeStartupTimeout"].(string) - result.NodeNotReadyTimeout = mhc["nodeUnknownTimeout"].(string) - result.NodeUnknownTimeout = mhc["nodeNotReadyTimeout"].(string) - return &result, nil -} - -// getContainerRegistryUrl gets the required information from the CSE Server configuration RDE -func getContainerRegistryUrl(client *Client, cseVersion string) (string, error) { - currentCseVersion := supportedCseVersions[cseVersion] - - rdes, err := getRdesByName(client, "vmware", "VCDKEConfig", currentCseVersion[0], "vcdKeConfig") - if err != nil { - return "", fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", currentCseVersion[0], err) - } - if len(rdes) != 1 { - return "", fmt.Errorf("expected exactly one VCDKEConfig RDE but got %d", len(rdes)) - } - // TODO: Get the struct Type for this one profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]any) if !ok { - return "", fmt.Errorf("wrong format of VCDKEConfig, expected a 'profiles' array") + return result, fmt.Errorf("wrong format of VCDKEConfig, expected a 'profiles' array") } if len(profiles) != 1 { - return "", fmt.Errorf("wrong format of VCDKEConfig, expected a single 'profiles' element, got %d", len(profiles)) + return result, fmt.Errorf("wrong format of VCDKEConfig, expected a single 'profiles' element, got %d", len(profiles)) } // TODO: Check airgapped environments: https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.1.1a/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.1.1/GUID-F00BE796-B5F2-48F2-A012-546E2E694400.html - return fmt.Sprintf("%s/tkg", profiles[0].(map[string]any)["containerRegistryUrl"].(string)), nil + result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]any)["containerRegistryUrl"]) + + if isNodeHealthCheckActive { + mhc, ok := profiles[0].(map[string]any)["K8Config"].(map[string]any)["mhc"].(map[string]any) + if !ok { + return result, nil + } + result.MaxUnhealthyNodesPercentage = mhc["maxUnhealthyNodes"].(float64) + result.NodeStartupTimeout = mhc["nodeStartupTimeout"].(string) + result.NodeNotReadyTimeout = mhc["nodeUnknownTimeout"].(string) + result.NodeUnknownTimeout = mhc["nodeNotReadyTimeout"].(string) + } + + return result, nil } func getCseTemplate(cseVersion, templateName string) (string, error) { diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index dfcb3ee39..3217b5693 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -176,7 +176,7 @@ func cseAddWorkerPoolsInYaml(docs []map[string]any, inputs []CseWorkerPoolSettin return nil, nil } -func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName string, machineHealthCheck *cseMachineHealthCheckInternal) ([]map[string]any, error) { +func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName string, vcdKeConfig *vcdKeConfig) ([]map[string]any, error) { mhcPosition := -1 result := make([]map[string]any, len(yamlDocuments)) for i, d := range yamlDocuments { @@ -188,13 +188,13 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName if mhcPosition < 0 { // There is no MachineHealthCheck block - if machineHealthCheck == nil { + if vcdKeConfig == nil { // We don't want it neither, so nothing to do return result, nil } // We need to add the block to the slice of YAML documents - mhcYaml, err := generateMemoryHealthCheckYaml(machineHealthCheck, "4.2", clusterName) + mhcYaml, err := generateMemoryHealthCheckYaml(*vcdKeConfig, "4.2", clusterName) if err != nil { return nil, err } @@ -206,7 +206,7 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName result = append(result, mhc) } else { // There is a MachineHealthCheck block - if machineHealthCheck != nil { + if vcdKeConfig != nil { // We want it, but it is already there, so nothing to do // TODO: What happens in UI if the VCDKEConfig MHC values are changed, does it get reflected in the cluster? // If that's the case, we might need to update this value always @@ -273,11 +273,11 @@ func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateIn } if input.NodeHealthCheck != nil { - mhcSettings, err := getMachineHealthCheck(client, input.vcdKeConfigVersion, *input.NodeHealthCheck) + vcdKeConfig, err := getVcdKeConfig(client, input.vcdKeConfigVersion, *input.NodeHealthCheck) if err != nil { return "", err } - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, input.clusterName, mhcSettings) + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, input.clusterName, &vcdKeConfig) if err != nil { return "", err } diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index b031228b0..12d544751 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -275,7 +275,7 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { } // Enables Machine Health Check - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, &cseMachineHealthCheckInternal{ + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, &vcdKeConfig{ MaxUnhealthyNodesPercentage: 12, NodeStartupTimeout: "34", NodeNotReadyTimeout: "56", diff --git a/types/v56/cse.go b/types/v56/cse.go index 55c2df2e1..1a17a9aa3 100644 --- a/types/v56/cse.go +++ b/types/v56/cse.go @@ -8,6 +8,9 @@ type Capvcd struct { Kind string `json:"kind"` Spec struct { VcdKe struct { + Secure struct { + ApiToken string `json:"apiToken"` + } `json:"secure"` IsVCDKECluster bool `json:"isVCDKECluster"` AutoRepairOnErrors bool `json:"autoRepairOnErrors"` DefaultStorageClassOptions struct { @@ -73,6 +76,9 @@ type Capvcd struct { Capvcd struct { Uid string `json:"uid"` Phase string `json:"phase"` + Private struct { + KubeConfig string `json:"kubeConfig"` + } `json:"private"` Upgrade struct { Ready bool `json:"ready"` Current struct { From b2d7dcec19b876f41837390718f486a2ded7ab31 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 5 Feb 2024 12:51:30 +0100 Subject: [PATCH 017/115] Checkpoint, not tested. Read finished but may be unstable Signed-off-by: abarreiro --- govcd/cse.go | 74 ++++----- govcd/cse_type.go | 2 - govcd/cse_util.go | 282 +++++++++++++++++++++-------------- govcd/system.go | 15 ++ govcd/vdccomputepolicy_v2.go | 12 +- 5 files changed, 217 insertions(+), 168 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 679ad8e60..c150a55e8 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -20,22 +20,26 @@ var supportedCseVersions = cseVersions{ } // CseCreateKubernetesCluster creates a Kubernetes cluster with the data given as input (CseClusterSettings). If the given -// timeout is 0, it waits forever for the cluster creation. Otherwise, if the timeout is reached and the cluster is not available -// (in "provisioned" state), it will return an error (the cluster will be left in VCD in any state) and the latest status -// of the cluster in the returned CseKubernetesCluster. -// If the cluster is created correctly, returns all the data in CseKubernetesCluster. +// timeout is 0, it waits forever for the cluster creation. +// +// If the timeout is reached and the cluster is not available (in "provisioned" state), it will return a non-nil CseKubernetesCluster +// with only the cluster ID and an error. This means that the cluster will be left in VCD in any state, and it can be retrieved with +// Org.CseGetKubernetesClusterById manually. +// +// If the cluster is created correctly, returns all the available data in CseKubernetesCluster or an error if some of the fields +// of the created cluster cannot be calculated or retrieved. func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterSettings, timeoutMinutes time.Duration) (*CseKubernetesCluster, error) { clusterId, err := org.CseCreateKubernetesClusterAsync(clusterData) if err != nil { return nil, err } - cluster, err := waitUntilClusterIsProvisioned(org.client, clusterId, timeoutMinutes) + err = waitUntilClusterIsProvisioned(org.client, clusterId, timeoutMinutes) if err != nil { - return cluster, err // Returns the latest status of the cluster + return &CseKubernetesCluster{ID: clusterId}, err } - return cluster, nil + return org.CseGetKubernetesClusterById(clusterId) } // CseCreateKubernetesClusterAsync creates a Kubernetes cluster with the data given as input (CseClusterSettings), but does not @@ -73,15 +77,26 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) // CseGetKubernetesClusterById retrieves a CSE Kubernetes cluster from VCD by its unique ID func (org *Org) CseGetKubernetesClusterById(id string) (*CseKubernetesCluster, error) { - rde, err := getRdeById(org.client, id) + return getCseKubernetesCluster(org.client, id) +} + +// getCseKubernetesCluster retrieves a CSE Kubernetes cluster from VCD by its unique ID +func getCseKubernetesCluster(client *Client, clusterId string) (*CseKubernetesCluster, error) { + rde, err := getRdeById(client, clusterId) if err != nil { return nil, err } - // This should be guaranteed by the proper rights, but just in case - if rde.DefinedEntity.Org.ID != org.Org.ID { - return nil, fmt.Errorf("could not find any Kubernetes cluster with ID '%s' in Organization '%s': %s", id, org.Org.Name, ErrorEntityNotFound) + return cseConvertToCseKubernetesClusterType(rde) +} + +// Refresh gets the latest information about the receiver cluster and updates its properties. +func (cluster *CseKubernetesCluster) Refresh() error { + refreshed, err := getCseKubernetesCluster(cluster.client, cluster.ID) + if err != nil { + return fmt.Errorf("failed refreshing the CSE Kubernetes Cluster: %s", err) } - return cseConvertToCseKubernetesClusterType(rde, nil) + *cluster = *refreshed + return nil } // GetKubeconfig retrieves the Kubeconfig from an available cluster. @@ -101,22 +116,6 @@ func (cluster *CseKubernetesCluster) GetKubeconfig() (string, error) { return capvcd.Status.Capvcd.Private.KubeConfig, nil } -// Refresh gets the latest information about the receiver cluster and updates its properties. -func (cluster *CseKubernetesCluster) Refresh() error { - rde, err := getRdeById(cluster.client, cluster.ID) - if err != nil { - return err - } - cluster.nameToIdCache = nil // We deliberately want to refresh everything - refreshed, err := cseConvertToCseKubernetesClusterType(rde, cluster.nameToIdCache) - if err != nil { - return err - } - cluster.capvcdType = refreshed.capvcdType - cluster.Etag = refreshed.Etag - return nil -} - // UpdateWorkerPools executes an update on the receiver cluster to change the existing worker pools. func (cluster *CseKubernetesCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput) error { return cluster.Update(CseClusterUpdateInput{ @@ -200,22 +199,13 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput) error { return err } - logHttpResponse := util.LogHttpResponse - // The following loop is constantly polling VCD to retrieve the RDE, which has a big JSON inside, so we avoid filling - // the log with these big payloads. We use defer to be sure that we restore the initial logging state. - defer func() { - util.LogHttpResponse = logHttpResponse - }() - // We do this loop to increase the chances that the Kubernetes cluster is successfully created, as the Go SDK is // "fighting" with the CSE Server retries := 0 maxRetries := 5 updated := false for retries <= maxRetries { - util.LogHttpResponse = false rde, err := getRdeById(cluster.client, cluster.ID) - util.LogHttpResponse = logHttpResponse if err != nil { return err } @@ -246,21 +236,11 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput) error { // Delete deletes a CSE Kubernetes cluster, waiting the specified amount of minutes. If the timeout is reached, this method // returns an error, even if the cluster is already marked for deletion. func (cluster *CseKubernetesCluster) Delete(timeoutMinutes time.Duration) error { - logHttpResponse := util.LogHttpResponse - - // The following loop is constantly polling VCD to retrieve the RDE, which has a big JSON inside, so we avoid filling - // the log with these big payloads. We use defer to be sure that we restore the initial logging state. - defer func() { - util.LogHttpResponse = logHttpResponse - }() - var elapsed time.Duration start := time.Now() vcdKe := map[string]interface{}{} for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies timeoutMinutes=0, we wait forever - util.LogHttpResponse = false rde, err := getRdeById(cluster.client, cluster.ID) - util.LogHttpResponse = logHttpResponse if err != nil { if ContainsNotFound(err) { return nil // The RDE is gone, so the process is completed and there's nothing more to do diff --git a/govcd/cse_type.go b/govcd/cse_type.go index 531bdb535..3548db5e1 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -22,8 +22,6 @@ type CseKubernetesCluster struct { client *Client capvcdType *types.Capvcd - - nameToIdCache map[string]string // This helps to reduce calls to VCD drastically, specially when doing item Name->ID transformations inside loops } // CseClusterEvent is an event that has occurred during the lifetime of a Container Service Extension (CSE) Kubernetes cluster. diff --git a/govcd/cse_util.go b/govcd/cse_util.go index b966adb1b..81630a0fb 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -16,8 +16,10 @@ import ( // and transforms it to an equivalent CseKubernetesCluster object that represents the same cluster, but // it is easy to explore and consume. If the input RDE is not a CSE Kubernetes cluster, this method // will obviously return an error. -// The nameToIdCache maps names with their IDs. This is used to reduce calls to VCD to retrieve this information. -func cseConvertToCseKubernetesClusterType(rde *DefinedEntity, nameToIdCache map[string]string) (*CseKubernetesCluster, error) { +// +// WARNING: Don't use this method inside loops or avoid calling it multiple times in a row, as it performs many queries +// to VCD. +func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesCluster, error) { requiredType := "vmware:capvcdCluster" if !strings.Contains(rde.DefinedEntity.ID, requiredType) || !strings.Contains(rde.DefinedEntity.EntityType, requiredType) { @@ -52,12 +54,9 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity, nameToIdCache map[ State: capvcd.Status.VcdKe.State, client: rde.client, capvcdType: capvcd, - nameToIdCache: nameToIdCache, - } - if result.nameToIdCache == nil { - result.nameToIdCache = map[string]string{} } + // Retrieve the Organization ID for i, binding := range capvcd.Status.Capvcd.ClusterResourceSetBindings { result.ClusterResourceSetBindings[i] = binding.ClusterResourceSetName } @@ -66,47 +65,65 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity, nameToIdCache map[ return nil, fmt.Errorf("could not read Organizations from Capvcd type") } result.OrganizationId = result.capvcdType.Status.Capvcd.VcdProperties.Organizations[0].Id + + // Retrieve the VDC ID if len(result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs) == 0 { return nil, fmt.Errorf("could not read VDCs from Capvcd type") } result.VdcId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Id - // To retrieve the Network ID, we check that it is not already cached. If it's not, we retrieve it with - // the VDC ID and name filters - if _, ok := result.nameToIdCache[result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName]; !ok { - params := url.Values{} - params.Add("filter", fmt.Sprintf("name==%s", result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName)) - params = queryParameterFilterAnd("ownerRef.id=="+result.VdcId, params) + // Retrieve the Network ID + params := url.Values{} + params.Add("filter", fmt.Sprintf("name==%s", result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName)) + params = queryParameterFilterAnd("ownerRef.id=="+result.VdcId, params) + networks, err := getAllOpenApiOrgVdcNetworks(rde.client, params) + if err != nil { + return nil, fmt.Errorf("could not read Org VDC Network from Capvcd type: %s", err) + } + if len(networks) != 1 { + return nil, fmt.Errorf("expected one Org VDC Network from Capvcd type, but got %d", len(networks)) + } + result.NetworkId = networks[0].OpenApiOrgVdcNetwork.ID - networks, err := getAllOpenApiOrgVdcNetworks(rde.client, params) - if err != nil { - return nil, fmt.Errorf("could not read Org VDC Network from Capvcd type: %s", err) + // Calculate the CSE Version + for cseVersion, subcomponents := range supportedCseVersions { + if subcomponents.CapvcdRdeTypeVersion == capvcd.Status.Capvcd.CapvcdVersion && + subcomponents.VcdKeConfigRdeTypeVersion == capvcd.Status.VcdKe.VcdKeVersion { + result.CseVersion = cseVersion + break } - if len(networks) != 1 { - return nil, fmt.Errorf("expected one Org VDC Network from Capvcd type, but got %d", len(networks)) - } - result.nameToIdCache[result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName] = networks[0].OpenApiOrgVdcNetwork.ID } - result.NetworkId = result.nameToIdCache[result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].OvdcNetworkName] + // Retrieve the Owner if rde.DefinedEntity.Owner == nil { return nil, fmt.Errorf("could not read Owner from RDE") } result.Owner = rde.DefinedEntity.Owner.Name + // Here we retrieve several items that we need from now onwards, like Storage Profiles and Compute Policies + storageProfiles, err := getAllStorageProfiles(rde.client) + if err != nil { + return nil, fmt.Errorf("could not get all the Storage Profiles: %s", err) + } + computePolicies, err := getAllVdcComputePoliciesV2(rde.client, nil) + if err != nil { + return nil, fmt.Errorf("could not get all the Compute Policies: %s", err) + } + if result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.K8SStorageClassName != "" { // This would mean there is a Default Storage Class defined result.DefaultStorageClass = &CseDefaultStorageClassSettings{ Name: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.K8SStorageClassName, - ReclaimPolicy: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName, - Filesystem: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName, + ReclaimPolicy: "retain", + Filesystem: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.Filesystem, } - - // To retrieve the Storage Profile ID, we check that it is not already cached. If it's not, we retrieve it - if _, ok := result.nameToIdCache[result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName]; !ok { - // TODO: There is no method to retrieve Storage profiles by name.... - result.nameToIdCache[result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName] = "" + for _, profile := range storageProfiles { + if profile.Name == result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName { + result.DefaultStorageClass.StorageProfileId = profile.ID + } + } + if result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.UseDeleteReclaimPolicy { + result.DefaultStorageClass.ReclaimPolicy = "delete" } - result.DefaultStorageClass.StorageProfileId = result.nameToIdCache[result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName] } yamlDocuments, err := unmarshalMultipleYamlDocuments(result.capvcdType.Spec.CapiYaml) @@ -114,91 +131,125 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity, nameToIdCache map[ return nil, err } - result.CseVersion = "" // TODO: Get opposite from supportedVersionsMap - - //var workerPools []CseWorkerPoolSettings + var workerPools []CseWorkerPoolSettings for _, yamlDocument := range yamlDocuments { switch yamlDocument["kind"] { case "KubeadmControlPlane": - // result.ControlPlane.MachineCount - // result.SshPublicKey + replicas, err := traverseMapAndGet[float64](yamlDocument, "spec.replicas") + if err != nil { + return nil, err + } + result.ControlPlane.MachineCount = int(replicas) + + users, err := traverseMapAndGet[[]any](yamlDocument, "spec.kubeadmConfigSpec.users") + if err != nil { + return nil, err + } + if len(users) == 0 { + return nil, fmt.Errorf("expected 'spec.kubeadmConfigSpec.users' slice to not to be empty") + } + keys, err := traverseMapAndGet[[]string](users[0], "sshAuthorizedKeys") + if err != nil { + return nil, err + } + if len(keys) == 0 { + return nil, fmt.Errorf("expected 'spec.kubeadmConfigSpec.users[0].sshAuthorizedKeys' slice to not to be empty") + } + result.SshPublicKey = keys[0] case "VCDMachineTemplate": - // Obtain all name->ID + sizingPolicyName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.sizingPolicy") + if err != nil || !strings.Contains(err.Error(), "key 'sizingPolicy' does not exist in input map") { + return nil, err + } + placementPolicyName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.placementPolicy") + if err != nil || !strings.Contains(err.Error(), "key 'placementPolicy' does not exist in input map") { + return nil, err + } + storageProfileName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.storageProfile") + if err != nil || !strings.Contains(err.Error(), "key 'storageProfile' does not exist in input map") { + return nil, err + } + diskSizeGi, err := traverseMapAndGet[float64](yamlDocument, "spec.template.spec.diskSize") + if err != nil { + return nil, err + } + if strings.Contains("name", "control-plane-node-pool") { + for _, policy := range computePolicies { + if sizingPolicyName == policy.VdcComputePolicyV2.Name && policy.VdcComputePolicyV2.IsSizingOnly { + result.ControlPlane.SizingPolicyId = policy.VdcComputePolicyV2.ID + } else if placementPolicyName == policy.VdcComputePolicyV2.Name && !policy.VdcComputePolicyV2.IsSizingOnly { + result.ControlPlane.PlacementPolicyId = policy.VdcComputePolicyV2.ID + } + } + for _, sp := range storageProfiles { + if storageProfileName == sp.Name { + result.ControlPlane.StorageProfileId = sp.ID + } + } + result.ControlPlane.DiskSizeGi = int(diskSizeGi) + + // We do it just once for the Control Plane because all VCDMachineTemplate blocks share the same OVA + ovaName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.template") + if err != nil { + return nil, err + } // TODO: There is no method to retrieve vApp templates by name.... - // result.KubernetesTemplateOvaId - // TODO: There is no method to retrieve vApp templates by name.... - // getAllVdcComputePoliciesV2() - // result.ControlPlane.SizingPolicyId - // result.ControlPlane.PlacementPolicyId - // result.ControlPlane.StorageProfileId - // result.ControlPlane.DiskSizeGi - fmt.Print("b") + result.KubernetesTemplateOvaId = ovaName } else { - fmt.Print("a") - //workerPool := CseWorkerPoolSettings{} - //workerPools = append(workerPools, workerPool) + workerPool := CseWorkerPoolSettings{} + + for _, policy := range computePolicies { + if sizingPolicyName == policy.VdcComputePolicyV2.Name && policy.VdcComputePolicyV2.IsSizingOnly { + workerPool.SizingPolicyId = policy.VdcComputePolicyV2.ID + } else if placementPolicyName == policy.VdcComputePolicyV2.Name && !policy.VdcComputePolicyV2.IsSizingOnly && !policy.VdcComputePolicyV2.IsVgpuPolicy { + workerPool.PlacementPolicyId = policy.VdcComputePolicyV2.ID + } else if placementPolicyName == policy.VdcComputePolicyV2.Name && !policy.VdcComputePolicyV2.IsSizingOnly && policy.VdcComputePolicyV2.IsVgpuPolicy { + workerPool.VGpuPolicyId = policy.VdcComputePolicyV2.ID + } + } + for _, sp := range storageProfiles { + if storageProfileName == sp.Name { + workerPool.StorageProfileId = sp.ID + } + } + workerPool.DiskSizeGi = int(diskSizeGi) + workerPools = append(workerPools, workerPool) } case "VCDCluster": - // result.ControlPlane.Ip - // result.VirtualIpSubnet + ip, err := traverseMapAndGet[string](yamlDocument, "spec.controlPlaneEndpoint.host") + if err != nil { + return nil, err + } + result.ControlPlane.Ip = ip + ip, err = traverseMapAndGet[string](yamlDocument, "spec.loadBalancerConfigSpec.vipSubnet") + if err == nil { + result.VirtualIpSubnet = ip // This is optional + } case "Cluster": - // result.PodCidr - // result.ServicesCidr + cidrBlocks, err := traverseMapAndGet[[]string](yamlDocument, "spec.clusterNetwork.pods.cidrBlocks") + if err != nil { + return nil, err + } + if len(cidrBlocks) == 0 { + return nil, fmt.Errorf("expected at least one 'spec.clusterNetwork.pods.cidrBlocks' item") + } + result.PodCidr = cidrBlocks[0] + + cidrBlocks, err = traverseMapAndGet[[]string](yamlDocument, "spec.clusterNetwork.services.cidrBlocks") + if err != nil { + return nil, err + } + if len(cidrBlocks) == 0 { + return nil, fmt.Errorf("expected at least one 'spec.clusterNetwork.services.cidrBlocks' item") + } + result.ServiceCidr = cidrBlocks[0] case "MachineHealthCheck": // This is quite simple, if we find this document, means that Machine Health Check is enabled result.NodeHealthCheck = true } } - - // // TODO: This needs a refactoring - // if nodePool.PlacementPolicy != "" { - // policies, err := vcdClient.GetAllVdcComputePoliciesV2(url.Values{ - // "filter": []string{fmt.Sprintf("name==%s", nodePool.PlacementPolicy)}, - // }) - // if err != nil { - // return nil, err // TODO - // } - // nameToIds[nodePool.PlacementPolicy] = policies[0].VdcComputePolicyV2.ID - // } - // if nodePool.SizingPolicy != "" { - // policies, err := vcdClient.GetAllVdcComputePoliciesV2(url.Values{ - // "filter": []string{fmt.Sprintf("name==%s", nodePool.SizingPolicy)}, - // }) - // if err != nil { - // return nil, err // TODO - // } - // nameToIds[nodePool.SizingPolicy] = policies[0].VdcComputePolicyV2.ID - // } - // if nodePool.StorageProfile != "" { - // ref, err := vdc.FindStorageProfileReference(nodePool.StorageProfile) - // if err != nil { - // return nil, fmt.Errorf("could not get Default Storage Class options from 'spec.vcdKe.defaultStorageClassOptions': %s", err) // TODO - // } - // nameToIds[nodePool.StorageProfile] = ref.ID - // } - // block["sizing_policy_id"] = nameToIds[nodePool.SizingPolicy] - // if nodePool.NvidiaGpuEnabled { // TODO: Be sure this is a worker node pool and not control plane (doesnt have this attr) - // block["vgpu_policy_id"] = nameToIds[nodePool.PlacementPolicy] // It's a placement policy here - // } else { - // block["placement_policy_id"] = nameToIds[nodePool.PlacementPolicy] - // } - // block["storage_profile_id"] = nameToIds[nodePool.StorageProfile] - // block["disk_size_gi"] = nodePool.DiskSizeMb / 1024 - // - // if strings.HasSuffix(nodePool.Name, "-control-plane-node-pool") { - // // Control Plane - // if len(cluster.Capvcd.Status.Capvcd.ClusterApiStatus.ApiEndpoints) == 0 { - // return nil, fmt.Errorf("could not retrieve Cluster IP") - // } - // block["ip"] = cluster.Capvcd.Status.Capvcd.ClusterApiStatus.ApiEndpoints[0].Host - // controlPlaneBlocks[0] = block - // } else { - // // Worker node - // block["name"] = nodePool.Name - // - // nodePoolBlocks[i] = block - // } + result.WorkerPools = workerPools return result, nil } @@ -209,45 +260,50 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity, nameToIdCache map[ // If one of the states of the cluster at a given point is "error", this function also checks whether the cluster has the "Auto Repair on Errors" flag enabled, // so it keeps waiting if it's true. // If timeout is reached before the cluster is in "provisioned" state, it returns an error. -func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinutes time.Duration) (*CseKubernetesCluster, error) { +func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinutes time.Duration) error { var elapsed time.Duration sleepTime := 10 start := time.Now() - cluster := &CseKubernetesCluster{} + capvcd := &types.Capvcd{} for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies timeoutMinutes=0, we wait forever rde, err := getRdeById(client, clusterId) if err != nil { - return nil, err + return err } - cluster, err = cseConvertToCseKubernetesClusterType(rde, cluster.nameToIdCache) + // Here we don't to use cseConvertToCseKubernetesClusterType to avoid calling VCD + entityBytes, err := json.Marshal(rde.DefinedEntity.Entity) if err != nil { - return nil, err + return fmt.Errorf("could not marshal the RDE contents to create a capvcdType instance: %s", err) } - switch cluster.State { + err = json.Unmarshal(entityBytes, &capvcd) + if err != nil { + return fmt.Errorf("could not unmarshal the RDE contents to create a Capvcd instance: %s", err) + } + + switch capvcd.Status.VcdKe.State { case "provisioned": - return cluster, nil + return nil case "error": // We just finish if auto-recovery is disabled, otherwise we just let CSE fixing things in background - if !cluster.AutoRepairOnErrors { + if !capvcd.Spec.VcdKe.AutoRepairOnErrors { // Give feedback about what went wrong errors := "" - for _, event := range cluster.Events { - if event.Type == "error" { - errors += fmt.Sprintf("%s\n", event.Details) - } + // TODO: Change to ErrorSet + for _, event := range capvcd.Status.Capvcd.EventSet { + errors += fmt.Sprintf("%s,\n", event.AdditionalDetails) } - return cluster, fmt.Errorf("got an error and 'auto repair on errors' is disabled, aborting. Errors:\n%s", errors) + return fmt.Errorf("got an error and 'AutoRepairOnErrors' is disabled, aborting. Errors:\n%s", errors) } } - util.Logger.Printf("[DEBUG] Cluster '%s' is in '%s' state, will check again in %d seconds", cluster.ID, cluster.State, sleepTime) + util.Logger.Printf("[DEBUG] Cluster '%s' is in '%s' state, will check again in %d seconds", rde.DefinedEntity.ID, capvcd.Status.VcdKe.State, sleepTime) elapsed = time.Since(start) time.Sleep(time.Duration(sleepTime) * time.Second) } - return cluster, fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, cluster.State) + return fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, capvcd.Status.VcdKe.State) } // validate validates the CSE Kubernetes cluster creation input data. Returns an error if some of the fields is wrong. diff --git a/govcd/system.go b/govcd/system.go index d1eef83e4..083d1c637 100644 --- a/govcd/system.go +++ b/govcd/system.go @@ -743,6 +743,21 @@ func getStorageProfileById(client *Client, id string) (*types.VdcStorageProfile, return vdcStorageProfile, nil } +// getAllStorageProfiles fetches all VDC Storage Profiles available to use. +func getAllStorageProfiles(client *Client) ([]*types.VdcStorageProfile, error) { + storageProfileHref := client.VCDHREF + storageProfileHref.Path += "/admin/vdcStorageProfile" + + vdcStorageProfiles := []*types.VdcStorageProfile{{}} + + _, err := client.ExecuteRequest(storageProfileHref.String(), http.MethodGet, "", "error retrieving all storage profiles: %s", nil, vdcStorageProfiles) + if err != nil { + return nil, err + } + + return vdcStorageProfiles, nil +} + // GetStorageProfileByHref fetches storage profile using provided HREF. // Deprecated: use client.GetStorageProfileByHref or vcdClient.GetStorageProfileByHref func GetStorageProfileByHref(vcdClient *VCDClient, url string) (*types.VdcStorageProfile, error) { diff --git a/govcd/vdccomputepolicy_v2.go b/govcd/vdccomputepolicy_v2.go index 3d0e67d43..68cc47a9b 100644 --- a/govcd/vdccomputepolicy_v2.go +++ b/govcd/vdccomputepolicy_v2.go @@ -61,26 +61,26 @@ func getVdcComputePolicyV2ById(client *Client, id string) (*VdcComputePolicyV2, // GetAllVdcComputePoliciesV2 retrieves all VDC Compute Policies (V2) using OpenAPI endpoint. Query parameters can be supplied to perform additional // filtering func (client *VCDClient) GetAllVdcComputePoliciesV2(queryParameters url.Values) ([]*VdcComputePolicyV2, error) { - return getAllVdcComputePoliciesV2(client, queryParameters) + return getAllVdcComputePoliciesV2(&client.Client, queryParameters) } // getAllVdcComputePolicies retrieves all VDC Compute Policies (V2) using OpenAPI endpoint. Query parameters can be supplied to perform additional // filtering -func getAllVdcComputePoliciesV2(client *VCDClient, queryParameters url.Values) ([]*VdcComputePolicyV2, error) { +func getAllVdcComputePoliciesV2(client *Client, queryParameters url.Values) ([]*VdcComputePolicyV2, error) { endpoint := types.OpenApiPathVersion2_0_0 + types.OpenApiEndpointVdcComputePolicies - minimumApiVersion, err := client.Client.checkOpenApiEndpointCompatibility(endpoint) + minimumApiVersion, err := client.checkOpenApiEndpointCompatibility(endpoint) if err != nil { return nil, err } - urlRef, err := client.Client.OpenApiBuildEndpoint(endpoint) + urlRef, err := client.OpenApiBuildEndpoint(endpoint) if err != nil { return nil, err } responses := []*types.VdcComputePolicyV2{{}} - err = client.Client.OpenApiGetAllItems(minimumApiVersion, urlRef, queryParameters, &responses, nil) + err = client.OpenApiGetAllItems(minimumApiVersion, urlRef, queryParameters, &responses, nil) if err != nil { return nil, err } @@ -88,7 +88,7 @@ func getAllVdcComputePoliciesV2(client *VCDClient, queryParameters url.Values) ( var wrappedVdcComputePolicies []*VdcComputePolicyV2 for _, response := range responses { wrappedVdcComputePolicy := &VdcComputePolicyV2{ - client: &client.Client, + client: client, VdcComputePolicyV2: response, } wrappedVdcComputePolicies = append(wrappedVdcComputePolicies, wrappedVdcComputePolicy) From 124cd62a7c57fa45ad99aa5eb20d3b3bc9d1439c Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 5 Feb 2024 18:14:05 +0100 Subject: [PATCH 018/115] Fixes, read works, get kubeconfig works Signed-off-by: abarreiro --- govcd/cse.go | 82 +++--- govcd/cse/{4.2 => 4.1}/capiyaml_cluster.tmpl | 0 govcd/cse/{4.2 => 4.1}/capiyaml_mhc.tmpl | 0 .../cse/{4.2 => 4.1}/capiyaml_workerpool.tmpl | 0 govcd/cse/{4.2 => 4.1}/rde.tmpl | 0 govcd/cse_template.go | 3 +- govcd/cse_test.go | 22 +- govcd/cse_type.go | 23 +- govcd/cse_util.go | 247 +++++++++++++----- govcd/cse_yaml.go | 7 +- govcd/cse_yaml_unit_test.go | 10 +- govcd/system.go | 47 ++-- 12 files changed, 298 insertions(+), 143 deletions(-) rename govcd/cse/{4.2 => 4.1}/capiyaml_cluster.tmpl (100%) rename govcd/cse/{4.2 => 4.1}/capiyaml_mhc.tmpl (100%) rename govcd/cse/{4.2 => 4.1}/capiyaml_workerpool.tmpl (100%) rename govcd/cse/{4.2 => 4.1}/rde.tmpl (100%) diff --git a/govcd/cse.go b/govcd/cse.go index c150a55e8..27ee3bf90 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -9,16 +9,6 @@ import ( "time" ) -// supportedCseVersions is a map that contains only the supported CSE versions with the versions of its subcomponents. -// TODO: Is this really necessary? What happens in UI if I have a 1.1.0-1.2.0-1.0.0 (4.2) cluster and then CSE is updated to 4.3? -var supportedCseVersions = cseVersions{ - "4.2": { - VcdKeConfigRdeTypeVersion: "1.1.0", - CapvcdRdeTypeVersion: "1.2.0", - CseInterfaceVersion: "1.0.0", - }, -} - // CseCreateKubernetesCluster creates a Kubernetes cluster with the data given as input (CseClusterSettings). If the given // timeout is 0, it waits forever for the cluster creation. // @@ -60,7 +50,12 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) return "", err } - rde, err := createRdeAndPoll(org.client, "vmware", "capvcdCluster", supportedCseVersions[clusterData.CseVersion].CapvcdRdeTypeVersion, types.DefinedEntity{ + cseSubcomponents, err := getCseComponentsVersions(clusterData.CseVersion) + if err != nil { + return "", err + } + + rde, err := createRdeAndPoll(org.client, "vmware", "capvcdCluster", cseSubcomponents.CapvcdRdeTypeVersion, types.DefinedEntity{ EntityType: goTemplateContents.RdeType.ID, Name: goTemplateContents.Name, Entity: rdeContents, @@ -105,68 +100,82 @@ func (cluster *CseKubernetesCluster) GetKubeconfig() (string, error) { if err != nil { return "", err } - var capvcd types.Capvcd - err = rde.InvokeBehaviorAndMarshal("", types.BehaviorInvocation{}, &capvcd) + versions, err := getCseComponentsVersions(cluster.CseVersion) + if err != nil { + return "", err + } + + // Auxiliary wrapper of the result, as the invocation returns the full RDE. + type invocationResult struct { + Capvcd types.Capvcd `json:"entity,omitempty"` + } + result := invocationResult{} + err = rde.InvokeBehaviorAndMarshal(fmt.Sprintf("urn:vcloud:behavior-interface:getFullEntity:cse:capvcd:%s", versions.CseInterfaceVersion), types.BehaviorInvocation{}, &result) if err != nil { return "", err } - if capvcd.Status.Capvcd.Private.KubeConfig == "" { + if result.Capvcd.Status.Capvcd.Private.KubeConfig == "" { return "", fmt.Errorf("could not retrieve the Kubeconfig from the invocation of the Behavior") } - return capvcd.Status.Capvcd.Private.KubeConfig, nil + return result.Capvcd.Status.Capvcd.Private.KubeConfig, nil } // UpdateWorkerPools executes an update on the receiver cluster to change the existing worker pools. -func (cluster *CseKubernetesCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput) error { +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +func (cluster *CseKubernetesCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ WorkerPools: &input, - }) + }, refresh) } // AddWorkerPools executes an update on the receiver cluster to add new worker pools. -func (cluster *CseKubernetesCluster) AddWorkerPools(input []CseWorkerPoolSettings) error { +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +func (cluster *CseKubernetesCluster) AddWorkerPools(input []CseWorkerPoolSettings, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ NewWorkerPools: &input, - }) + }, refresh) } // UpdateControlPlane executes an update on the receiver cluster to change the existing control plane. -func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpdateInput) error { +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpdateInput, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ ControlPlane: &input, - }) + }, refresh) } // ChangeKubernetesTemplate executes an update on the receiver cluster to change the Kubernetes template of the cluster. -func (cluster *CseKubernetesCluster) ChangeKubernetesTemplate(kubernetesTemplateOvaId string) error { +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +func (cluster *CseKubernetesCluster) ChangeKubernetesTemplate(kubernetesTemplateOvaId string, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ KubernetesTemplateOvaId: &kubernetesTemplateOvaId, - }) + }, refresh) } // SetHealthCheck executes an update on the receiver cluster to enable or disable the machine health check capabilities. -func (cluster *CseKubernetesCluster) SetHealthCheck(healthCheckEnabled bool) error { +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +func (cluster *CseKubernetesCluster) SetHealthCheck(healthCheckEnabled bool, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ NodeHealthCheck: &healthCheckEnabled, - }) + }, refresh) } // SetAutoRepairOnErrors executes an update on the receiver cluster to change the flag that controls the auto-repair -// capabilities of CSE. -func (cluster *CseKubernetesCluster) SetAutoRepairOnErrors(autoRepairOnErrors bool) error { +// capabilities of CSE. If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +func (cluster *CseKubernetesCluster) SetAutoRepairOnErrors(autoRepairOnErrors bool, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ AutoRepairOnErrors: &autoRepairOnErrors, - }) + }, refresh) } -// Update executes a synchronous update on the receiver cluster to perform a update on any of the allowed parameters of the cluster. If the given -// timeout is 0, it waits forever for the cluster update to finish. Otherwise, if the timeout is reached and the cluster is not available, -// it will return an error (the cluster will be left in VCD in any state) and the latest status of the cluster will be available in the -// receiver CseKubernetesCluster. -func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput) error { - err := cluster.Refresh() - if err != nil { - return err +// Update executes an update on the receiver CSE Kubernetes Cluster on any of the allowed parameters defined in the input type. If refresh=true, +// it retrieves the latest state of the cluster from VCD before updating (recommended). +func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh bool) error { + if refresh { + err := cluster.Refresh() + if err != nil { + return err + } } if cluster.capvcdType.Status.VcdKe.State == "" { @@ -183,6 +192,7 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput) error { // Computed attributes that are required, such as the VcdKeConfig version input.clusterName = cluster.Name input.vcdKeConfigVersion = cluster.capvcdType.Status.VcdKe.VcdKeVersion + input.cseVersion = cluster.CseVersion updatedCapiYaml, err := cseUpdateCapiYaml(cluster.client, cluster.capvcdType.Spec.CapiYaml, input) if err != nil { return err diff --git a/govcd/cse/4.2/capiyaml_cluster.tmpl b/govcd/cse/4.1/capiyaml_cluster.tmpl similarity index 100% rename from govcd/cse/4.2/capiyaml_cluster.tmpl rename to govcd/cse/4.1/capiyaml_cluster.tmpl diff --git a/govcd/cse/4.2/capiyaml_mhc.tmpl b/govcd/cse/4.1/capiyaml_mhc.tmpl similarity index 100% rename from govcd/cse/4.2/capiyaml_mhc.tmpl rename to govcd/cse/4.1/capiyaml_mhc.tmpl diff --git a/govcd/cse/4.2/capiyaml_workerpool.tmpl b/govcd/cse/4.1/capiyaml_workerpool.tmpl similarity index 100% rename from govcd/cse/4.2/capiyaml_workerpool.tmpl rename to govcd/cse/4.1/capiyaml_workerpool.tmpl diff --git a/govcd/cse/4.2/rde.tmpl b/govcd/cse/4.1/rde.tmpl similarity index 100% rename from govcd/cse/4.2/rde.tmpl rename to govcd/cse/4.1/rde.tmpl diff --git a/govcd/cse_template.go b/govcd/cse_template.go index fbf144deb..bdc92b533 100644 --- a/govcd/cse_template.go +++ b/govcd/cse_template.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + semver "github.com/hashicorp/go-version" "strconv" "strings" "text/template" @@ -105,7 +106,7 @@ func generateNodePoolYaml(clusterDetails cseClusterSettingsInternal) (string, er } // generateMemoryHealthCheckYaml generates a YAML block corresponding to the Kubernetes memory health check. -func generateMemoryHealthCheckYaml(vcdKeConfig vcdKeConfig, cseVersion, clusterName string) (string, error) { +func generateMemoryHealthCheckYaml(vcdKeConfig vcdKeConfig, cseVersion semver.Version, clusterName string) (string, error) { if vcdKeConfig.NodeStartupTimeout == "" && vcdKeConfig.NodeUnknownTimeout == "" && vcdKeConfig.NodeNotReadyTimeout == "" && vcdKeConfig.MaxUnhealthyNodesPercentage == 0 { return "", nil diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 5bdba8214..eafce1bd5 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -8,6 +8,7 @@ package govcd import ( "fmt" + semver "github.com/hashicorp/go-version" "github.com/vmware/go-vcloud-director/v2/types/v56" . "gopkg.in/check.v1" "net/url" @@ -67,13 +68,17 @@ func (vcd *TestVCD) Test_Cse(check *C) { apiToken, err := token.GetInitialApiToken() check.Assert(err, IsNil) + cseVersion, err := semver.NewVersion("4.1") + check.Assert(err, IsNil) + check.Assert(cseVersion, NotNil) + clusterSettings := CseClusterSettings{ Name: "test-cse", OrganizationId: org.Org.ID, VdcId: vdc.Vdc.ID, NetworkId: net.OrgVDCNetwork.ID, KubernetesTemplateOvaId: ova.VAppTemplate.ID, - CseVersion: "4.2", + CseVersion: *cseVersion, ControlPlane: CseControlPlaneSettings{ MachineCount: 1, DiskSizeGi: 20, @@ -142,7 +147,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { } } // Perform the update - err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{clusterSettings.WorkerPools[0].Name: {MachineCount: 2}}) + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{clusterSettings.WorkerPools[0].Name: {MachineCount: 2}}, true) check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up @@ -172,16 +177,23 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { workerPoolName := "cse-test1-worker-node-pool-1" + kubeconfig, err := cluster.GetKubeconfig() + check.Assert(err, IsNil) + check.Assert(true, Equals, strings.Contains(kubeconfig, cluster.Name)) + check.Assert(true, Equals, strings.Contains(kubeconfig, "client-certificate-data")) + check.Assert(true, Equals, strings.Contains(kubeconfig, "certificate-authority-data")) + check.Assert(true, Equals, strings.Contains(kubeconfig, "client-key-data")) + // Perform the update - err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}) + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 1}}, true) check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up foundWorkerPool := false - for _, nodePool := range cluster.capvcdType.Status.Capvcd.NodePool { + for _, nodePool := range cluster.WorkerPools { if nodePool.Name == workerPoolName { foundWorkerPool = true - check.Assert(nodePool.DesiredReplicas, Equals, 2) + check.Assert(nodePool.MachineCount, Equals, 1) } } check.Assert(foundWorkerPool, Equals, true) diff --git a/govcd/cse_type.go b/govcd/cse_type.go index 3548db5e1..b634e4efd 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -2,6 +2,7 @@ package govcd import ( "embed" + semver "github.com/hashicorp/go-version" "github.com/vmware/go-vcloud-director/v2/types/v56" "time" ) @@ -11,12 +12,12 @@ type CseKubernetesCluster struct { CseClusterSettings ID string Etag string - KubernetesVersion string - TkgVersion string - CapvcdVersion string + KubernetesVersion semver.Version + TkgVersion semver.Version + CapvcdVersion semver.Version ClusterResourceSetBindings []string - CpiVersion string - CsiVersion string + CpiVersion semver.Version + CsiVersion semver.Version State string Events []CseClusterEvent @@ -34,12 +35,12 @@ type CseClusterEvent struct { // CseClusterSettings defines the required configuration of a Container Service Extension (CSE) Kubernetes cluster. type CseClusterSettings struct { + CseVersion semver.Version Name string OrganizationId string VdcId string NetworkId string KubernetesTemplateOvaId string - CseVersion string ControlPlane CseControlPlaneSettings WorkerPools []CseWorkerPoolSettings DefaultStorageClass *CseDefaultStorageClassSettings // Optional @@ -94,6 +95,7 @@ type CseClusterUpdateInput struct { // Private fields that are computed, not requested to the consumer of this struct vcdKeConfigVersion string clusterName string + cseVersion semver.Version } // CseControlPlaneUpdateInput defines the required configuration that the Control Plane of the Container Service Extension (CSE) Kubernetes cluster @@ -116,7 +118,7 @@ type CseWorkerPoolUpdateInput struct { // The main difference between CseClusterSettings and this structure is that the first one uses IDs and this one uses names, among // other differences like the computed TkgVersionBundle. type cseClusterSettingsInternal struct { - CseVersion string + CseVersion semver.Version Name string OrganizationName string VdcName string @@ -178,16 +180,13 @@ type vcdKeConfig struct { ContainerRegistryUrl string } -// cseComponentVersions is a type that registers the versions of the subcomponents of a specific CSE Version -type cseComponentVersions struct { +// cseComponentsVersions is a type that registers the versions of the subcomponents of a specific CSE Version +type cseComponentsVersions struct { VcdKeConfigRdeTypeVersion string CapvcdRdeTypeVersion string CseInterfaceVersion string } -// cseVersions is a map that links a CSE Version with the versions of its subcomponents -type cseVersions map[string]cseComponentVersions - // This collection of files contains all the Go Templates and resources required for the Container Service Extension (CSE) methods // to work. // diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 81630a0fb..f8da5ab0b 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -4,14 +4,31 @@ import ( _ "embed" "encoding/json" "fmt" + semver "github.com/hashicorp/go-version" "github.com/vmware/go-vcloud-director/v2/types/v56" "github.com/vmware/go-vcloud-director/v2/util" "net/url" "regexp" + "strconv" "strings" "time" ) +// getCseComponentsVersions gets the CSE components versions from its version. +// TODO: Is this really necessary? What happens in UI if I have a 1.1.0-1.2.0-1.0.0 (4.2) cluster and then CSE is updated to 4.3? +func getCseComponentsVersions(cseVersion semver.Version) (*cseComponentsVersions, error) { + v42, _ := semver.NewVersion("4.1") + + if cseVersion.Equal(v42) { + return &cseComponentsVersions{ + VcdKeConfigRdeTypeVersion: "1.1.0", + CapvcdRdeTypeVersion: "1.2.0", + CseInterfaceVersion: "1.0.0", + }, nil + } + return nil, fmt.Errorf("not supported version %s", cseVersion.String()) +} + // cseConvertToCseKubernetesClusterType takes a generic RDE that must represent an existing CSE Kubernetes cluster, // and transforms it to an equivalent CseKubernetesCluster object that represents the same cluster, but // it is easy to explore and consume. If the input RDE is not a CSE Kubernetes cluster, this method @@ -42,24 +59,54 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu Name: rde.DefinedEntity.Name, ApiToken: "******", // We can't return this one, we return the "standard" 6-asterisk value AutoRepairOnErrors: capvcd.Spec.VcdKe.AutoRepairOnErrors, + ControlPlane: CseControlPlaneSettings{}, }, ID: rde.DefinedEntity.ID, Etag: rde.Etag, - KubernetesVersion: capvcd.Status.Capvcd.Upgrade.Current.KubernetesVersion, - TkgVersion: capvcd.Status.Capvcd.Upgrade.Current.TkgVersion, - CapvcdVersion: capvcd.Status.Capvcd.CapvcdVersion, ClusterResourceSetBindings: make([]string, len(capvcd.Status.Capvcd.ClusterResourceSetBindings)), - CpiVersion: capvcd.Status.Cpi.Version, - CsiVersion: capvcd.Status.Csi.Version, State: capvcd.Status.VcdKe.State, client: rde.client, capvcdType: capvcd, } + version, err := semver.NewVersion(capvcd.Status.Capvcd.Upgrade.Current.KubernetesVersion) + if err != nil { + return nil, fmt.Errorf("could not read Kubernetes version: %s", err) + } + result.KubernetesVersion = *version + + version, err = semver.NewVersion(capvcd.Status.Capvcd.Upgrade.Current.TkgVersion) + if err != nil { + return nil, fmt.Errorf("could not read Tkg version: %s", err) + } + result.TkgVersion = *version + + version, err = semver.NewVersion(capvcd.Status.Capvcd.CapvcdVersion) + if err != nil { + return nil, fmt.Errorf("could not read Capvcd version: %s", err) + } + result.CapvcdVersion = *version + + version, err = semver.NewVersion(strings.TrimSpace(capvcd.Status.Cpi.Version)) // Note: We use trim as the version comes with spacing characters + if err != nil { + return nil, fmt.Errorf("could not read CPI version: %s", err) + } + result.CpiVersion = *version + + version, err = semver.NewVersion(capvcd.Status.Csi.Version) + if err != nil { + return nil, fmt.Errorf("could not read CSI version: %s", err) + } + result.CsiVersion = *version // Retrieve the Organization ID for i, binding := range capvcd.Status.Capvcd.ClusterResourceSetBindings { - result.ClusterResourceSetBindings[i] = binding.ClusterResourceSetName + result.ClusterResourceSetBindings[i] = binding.Name + } + + if len(capvcd.Status.Capvcd.ClusterApiStatus.ApiEndpoints) == 0 { + return nil, fmt.Errorf("could not get Control Plane endpoint") } + result.ControlPlane.Ip = capvcd.Status.Capvcd.ClusterApiStatus.ApiEndpoints[0].Host if len(result.capvcdType.Status.Capvcd.VcdProperties.Organizations) == 0 { return nil, fmt.Errorf("could not read Organizations from Capvcd type") @@ -70,7 +117,26 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu if len(result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs) == 0 { return nil, fmt.Errorf("could not read VDCs from Capvcd type") } + // FIXME: This is a workaround, because for some reason the ID contains the VDC name instead of the VDC ID. + // Once this is fixed, this conditional should not be needed anymore. result.VdcId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Id + if result.VdcId == result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Name { + vdcs, err := queryOrgVdcList(rde.client, map[string]string{}) + if err != nil { + return nil, fmt.Errorf("could not get VDC IDs as no VDC was found: %s", err) + } + found := false + for _, vdc := range vdcs { + if vdc.Name == result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Name { + result.VdcId = fmt.Sprintf("urn:vcloud:vdc:%s", extractUuid(vdc.HREF)) + found = true + break + } + } + if !found { + return nil, fmt.Errorf("could not get VDC IDs as no VDC with name '%s' was found", result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Name) + } + } // Retrieve the Network ID params := url.Values{} @@ -85,14 +151,12 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu } result.NetworkId = networks[0].OpenApiOrgVdcNetwork.ID - // Calculate the CSE Version - for cseVersion, subcomponents := range supportedCseVersions { - if subcomponents.CapvcdRdeTypeVersion == capvcd.Status.Capvcd.CapvcdVersion && - subcomponents.VcdKeConfigRdeTypeVersion == capvcd.Status.VcdKe.VcdKeVersion { - result.CseVersion = cseVersion - break - } + // Get the CSE version + cseVersion, err := semver.NewVersion(capvcd.Status.VcdKe.VcdKeVersion) + if err != nil { + return nil, fmt.Errorf("could not read the CSE Version that the cluster uses: %s", err) } + result.CseVersion = *cseVersion // Retrieve the Owner if rde.DefinedEntity.Owner == nil { @@ -101,10 +165,25 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu result.Owner = rde.DefinedEntity.Owner.Name // Here we retrieve several items that we need from now onwards, like Storage Profiles and Compute Policies - storageProfiles, err := getAllStorageProfiles(rde.client) - if err != nil { - return nil, fmt.Errorf("could not get all the Storage Profiles: %s", err) + storageProfiles := map[string]string{} + if rde.client.IsSysAdmin { + allSp, err := queryAdminOrgVdcStorageProfilesByVdcId(rde.client, result.VdcId) + if err != nil { + return nil, fmt.Errorf("could not get all the Storage Profiles: %s", err) + } + for _, recordType := range allSp { + storageProfiles[recordType.Name] = fmt.Sprintf("urn:vcloud:vdcstorageProfile:%s", extractUuid(recordType.HREF)) + } + } else { + allSp, err := queryOrgVdcStorageProfilesByVdcId(rde.client, result.VdcId) + if err != nil { + return nil, fmt.Errorf("could not get all the Storage Profiles: %s", err) + } + for _, recordType := range allSp { + storageProfiles[recordType.Name] = fmt.Sprintf("urn:vcloud:vdcstorageProfile:%s", extractUuid(recordType.HREF)) + } } + computePolicies, err := getAllVdcComputePoliciesV2(rde.client, nil) if err != nil { return nil, fmt.Errorf("could not get all the Compute Policies: %s", err) @@ -116,9 +195,9 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu ReclaimPolicy: "retain", Filesystem: result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.Filesystem, } - for _, profile := range storageProfiles { - if profile.Name == result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName { - result.DefaultStorageClass.StorageProfileId = profile.ID + for spName, spId := range storageProfiles { + if spName == result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.VcdStorageProfileName { + result.DefaultStorageClass.StorageProfileId = spId } } if result.capvcdType.Spec.VcdKe.DefaultStorageClassOptions.UseDeleteReclaimPolicy { @@ -131,7 +210,10 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu return nil, err } - var workerPools []CseWorkerPoolSettings + // We need a map of worker pools and not a slice, because there are two types of YAML documents + // that contain data about a specific worker pool (VCDMachineTemplate and MachineDeployment), and we can get them in no + // particular order, so we store the worker pools with their name as key. This way we can easily fetch them and override them. + workerPools := map[string]CseWorkerPoolSettings{} for _, yamlDocument := range yamlDocuments { switch yamlDocument["kind"] { case "KubeadmControlPlane": @@ -149,32 +231,40 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu return nil, fmt.Errorf("expected 'spec.kubeadmConfigSpec.users' slice to not to be empty") } keys, err := traverseMapAndGet[[]string](users[0], "sshAuthorizedKeys") - if err != nil { + if err != nil && !strings.Contains(err.Error(), "key 'sshAuthorizedKeys' does not exist in input map") { return nil, err } - if len(keys) == 0 { - return nil, fmt.Errorf("expected 'spec.kubeadmConfigSpec.users[0].sshAuthorizedKeys' slice to not to be empty") + if len(keys) > 0 { + result.SshPublicKey = keys[0] // Optional field } - result.SshPublicKey = keys[0] case "VCDMachineTemplate": + name, err := traverseMapAndGet[string](yamlDocument, "metadata.name") + if err != nil { + return nil, err + } sizingPolicyName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.sizingPolicy") - if err != nil || !strings.Contains(err.Error(), "key 'sizingPolicy' does not exist in input map") { + if err != nil && !strings.Contains(err.Error(), "key 'sizingPolicy' does not exist in input map") { return nil, err } placementPolicyName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.placementPolicy") - if err != nil || !strings.Contains(err.Error(), "key 'placementPolicy' does not exist in input map") { + if err != nil && !strings.Contains(err.Error(), "key 'placementPolicy' does not exist in input map") { return nil, err } storageProfileName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.storageProfile") - if err != nil || !strings.Contains(err.Error(), "key 'storageProfile' does not exist in input map") { + if err != nil && !strings.Contains(err.Error(), "key 'storageProfile' does not exist in input map") { + return nil, err + } + diskSizeGiRaw, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.diskSize") + if err != nil { return nil, err } - diskSizeGi, err := traverseMapAndGet[float64](yamlDocument, "spec.template.spec.diskSize") + diskSizeGi, err := strconv.Atoi(strings.ReplaceAll(diskSizeGiRaw, "Gi", "")) if err != nil { return nil, err } - if strings.Contains("name", "control-plane-node-pool") { + if strings.Contains(name, "control-plane-node-pool") { + // This is the single Control Plane for _, policy := range computePolicies { if sizingPolicyName == policy.VdcComputePolicyV2.Name && policy.VdcComputePolicyV2.IsSizingOnly { result.ControlPlane.SizingPolicyId = policy.VdcComputePolicyV2.ID @@ -182,12 +272,13 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu result.ControlPlane.PlacementPolicyId = policy.VdcComputePolicyV2.ID } } - for _, sp := range storageProfiles { - if storageProfileName == sp.Name { - result.ControlPlane.StorageProfileId = sp.ID + for spName, spId := range storageProfiles { + if storageProfileName == spName { + result.ControlPlane.StorageProfileId = spId } } - result.ControlPlane.DiskSizeGi = int(diskSizeGi) + + result.ControlPlane.DiskSizeGi = diskSizeGi // We do it just once for the Control Plane because all VCDMachineTemplate blocks share the same OVA ovaName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.template") @@ -197,8 +288,13 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu // TODO: There is no method to retrieve vApp templates by name.... result.KubernetesTemplateOvaId = ovaName } else { - workerPool := CseWorkerPoolSettings{} - + // This is one Worker Pool. We need to check the map of worker pools, just in case we already saved the + // machine count from MachineDeployment. + if _, ok := workerPools[name]; !ok { + workerPools[name] = CseWorkerPoolSettings{} + } + workerPool := workerPools[name] + workerPool.Name = name for _, policy := range computePolicies { if sizingPolicyName == policy.VdcComputePolicyV2.Name && policy.VdcComputePolicyV2.IsSizingOnly { workerPool.SizingPolicyId = policy.VdcComputePolicyV2.ID @@ -208,48 +304,65 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu workerPool.VGpuPolicyId = policy.VdcComputePolicyV2.ID } } - for _, sp := range storageProfiles { - if storageProfileName == sp.Name { - workerPool.StorageProfileId = sp.ID + for spName, spId := range storageProfiles { + if storageProfileName == spName { + workerPool.StorageProfileId = spId } } - workerPool.DiskSizeGi = int(diskSizeGi) - workerPools = append(workerPools, workerPool) + workerPool.DiskSizeGi = diskSizeGi + workerPools[name] = workerPool // Override the worker pool with the updated data } - case "VCDCluster": - ip, err := traverseMapAndGet[string](yamlDocument, "spec.controlPlaneEndpoint.host") + case "MachineDeployment": + name, err := traverseMapAndGet[string](yamlDocument, "metadata.name") + if err != nil { + return nil, err + } + // This is one Worker Pool. We need to check the map of worker pools, just in case we already saved the + // other information from VCDMachineTemplate. + if _, ok := workerPools[name]; !ok { + workerPools[name] = CseWorkerPoolSettings{} + } + workerPool := workerPools[name] + replicas, err := traverseMapAndGet[float64](yamlDocument, "spec.replicas") if err != nil { return nil, err } - result.ControlPlane.Ip = ip - ip, err = traverseMapAndGet[string](yamlDocument, "spec.loadBalancerConfigSpec.vipSubnet") + workerPool.MachineCount = int(replicas) + workerPools[name] = workerPool // Override the worker pool with the updated data + case "VCDCluster": + subnet, err := traverseMapAndGet[string](yamlDocument, "spec.loadBalancerConfigSpec.vipSubnet") if err == nil { - result.VirtualIpSubnet = ip // This is optional + result.VirtualIpSubnet = subnet // This is optional } case "Cluster": - cidrBlocks, err := traverseMapAndGet[[]string](yamlDocument, "spec.clusterNetwork.pods.cidrBlocks") + cidrBlocks, err := traverseMapAndGet[[]any](yamlDocument, "spec.clusterNetwork.pods.cidrBlocks") if err != nil { return nil, err } if len(cidrBlocks) == 0 { return nil, fmt.Errorf("expected at least one 'spec.clusterNetwork.pods.cidrBlocks' item") } - result.PodCidr = cidrBlocks[0] + result.PodCidr = cidrBlocks[0].(string) - cidrBlocks, err = traverseMapAndGet[[]string](yamlDocument, "spec.clusterNetwork.services.cidrBlocks") + cidrBlocks, err = traverseMapAndGet[[]any](yamlDocument, "spec.clusterNetwork.services.cidrBlocks") if err != nil { return nil, err } if len(cidrBlocks) == 0 { return nil, fmt.Errorf("expected at least one 'spec.clusterNetwork.services.cidrBlocks' item") } - result.ServiceCidr = cidrBlocks[0] + result.ServiceCidr = cidrBlocks[0].(string) case "MachineHealthCheck": // This is quite simple, if we find this document, means that Machine Health Check is enabled result.NodeHealthCheck = true } } - result.WorkerPools = workerPools + result.WorkerPools = make([]CseWorkerPoolSettings, len(workerPools)) + i := 0 + for _, workerPool := range workerPools { + result.WorkerPools[i] = workerPool + i++ + } return result, nil } @@ -329,8 +442,9 @@ func (input *CseClusterSettings) validate() error { if input.NetworkId == "" { return fmt.Errorf("the Network ID is required") } - if _, ok := supportedCseVersions[input.CseVersion]; !ok { - return fmt.Errorf("the CSE version '%s' is not supported. Must be one of %v", input.CseVersion, getKeys(supportedCseVersions)) + _, err = getCseComponentsVersions(input.CseVersion) + if err != nil { + return fmt.Errorf("the CSE version '%s' is not supported", input.CseVersion.String()) } if input.ControlPlane.MachineCount < 1 || input.ControlPlane.MachineCount%2 == 0 { return fmt.Errorf("number of control plane nodes must be odd and higher than 0, but it was '%d'", input.ControlPlane.MachineCount) @@ -424,9 +538,9 @@ func cseClusterSettingsToInternal(input CseClusterSettings, org Org) (cseCluster } output.NetworkName = network.OrgVDCNetwork.Name - currentCseVersion, ok := supportedCseVersions[input.CseVersion] - if !ok { - return output, fmt.Errorf("the CSE version '%s' is not supported. List of supported versions: %v", input.CseVersion, getKeys(supportedCseVersions)) + currentCseVersion, err := getCseComponentsVersions(input.CseVersion) + if err != nil { + return output, fmt.Errorf("the CSE version '%s' is not supported: %s", input.CseVersion.String(), err) } rdeType, err := getRdeType(org.client, "vmware", "capvcdCluster", currentCseVersion.CapvcdRdeTypeVersion) if err != nil { @@ -501,7 +615,12 @@ func cseClusterSettingsToInternal(input CseClusterSettings, org Org) (cseCluster } } - vcdKeConfig, err := getVcdKeConfig(org.client, supportedCseVersions[input.CseVersion].VcdKeConfigRdeTypeVersion, input.NodeHealthCheck) + cseVersions, err := getCseComponentsVersions(input.CseVersion) + if err != nil { + return output, err + } + + vcdKeConfig, err := getVcdKeConfig(org.client, cseVersions.VcdKeConfigRdeTypeVersion, input.NodeHealthCheck) if err != nil { return output, err } @@ -621,21 +740,11 @@ func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheck return result, nil } -func getCseTemplate(cseVersion, templateName string) (string, error) { - result, err := cseFiles.ReadFile(fmt.Sprintf("cse/%s/%s.tmpl", cseVersion, templateName)) +func getCseTemplate(cseVersion semver.Version, templateName string) (string, error) { + cseVersionSegments := cseVersion.Segments() + result, err := cseFiles.ReadFile(fmt.Sprintf("cse/%d.%d/%s.tmpl", cseVersionSegments[0], cseVersionSegments[1], templateName)) if err != nil { return "", err } return string(result), nil } - -// getKeys retrieves all the keys from the given map and returns them as a slice -func getKeys[K comparable, V any](input map[K]V) []K { - result := make([]K, len(input)) - i := 0 - for k := range input { - result[i] = k - i++ - } - return result -} diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 3217b5693..b2e9c66e2 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -2,6 +2,7 @@ package govcd import ( "fmt" + semver "github.com/hashicorp/go-version" "sigs.k8s.io/yaml" "strings" ) @@ -176,7 +177,7 @@ func cseAddWorkerPoolsInYaml(docs []map[string]any, inputs []CseWorkerPoolSettin return nil, nil } -func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName string, vcdKeConfig *vcdKeConfig) ([]map[string]any, error) { +func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName string, cseVersion semver.Version, vcdKeConfig *vcdKeConfig) ([]map[string]any, error) { mhcPosition := -1 result := make([]map[string]any, len(yamlDocuments)) for i, d := range yamlDocuments { @@ -194,7 +195,7 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName } // We need to add the block to the slice of YAML documents - mhcYaml, err := generateMemoryHealthCheckYaml(*vcdKeConfig, "4.2", clusterName) + mhcYaml, err := generateMemoryHealthCheckYaml(*vcdKeConfig, cseVersion, clusterName) if err != nil { return nil, err } @@ -277,7 +278,7 @@ func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateIn if err != nil { return "", err } - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, input.clusterName, &vcdKeConfig) + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, input.clusterName, input.cseVersion, &vcdKeConfig) if err != nil { return "", err } diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 12d544751..d3bc96436 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -3,6 +3,7 @@ package govcd import ( + semver "github.com/hashicorp/go-version" "os" "reflect" "strings" @@ -261,8 +262,13 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { t.Fatal("could not find the cluster name in the CAPI YAML test file") } + v, err := semver.NewVersion("4.1") + if err != nil { + t.Fatalf("incorrect version: %s", err) + } + // Deactivates Machine Health Check - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, nil) + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, nil) if err != nil { t.Fatalf("%s", err) } @@ -275,7 +281,7 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { } // Enables Machine Health Check - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, &vcdKeConfig{ + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, &vcdKeConfig{ MaxUnhealthyNodesPercentage: 12, NodeStartupTimeout: "34", NodeNotReadyTimeout: "56", diff --git a/govcd/system.go b/govcd/system.go index 083d1c637..e543854a9 100644 --- a/govcd/system.go +++ b/govcd/system.go @@ -743,21 +743,6 @@ func getStorageProfileById(client *Client, id string) (*types.VdcStorageProfile, return vdcStorageProfile, nil } -// getAllStorageProfiles fetches all VDC Storage Profiles available to use. -func getAllStorageProfiles(client *Client) ([]*types.VdcStorageProfile, error) { - storageProfileHref := client.VCDHREF - storageProfileHref.Path += "/admin/vdcStorageProfile" - - vdcStorageProfiles := []*types.VdcStorageProfile{{}} - - _, err := client.ExecuteRequest(storageProfileHref.String(), http.MethodGet, "", "error retrieving all storage profiles: %s", nil, vdcStorageProfiles) - if err != nil { - return nil, err - } - - return vdcStorageProfiles, nil -} - // GetStorageProfileByHref fetches storage profile using provided HREF. // Deprecated: use client.GetStorageProfileByHref or vcdClient.GetStorageProfileByHref func GetStorageProfileByHref(vcdClient *VCDClient, url string) (*types.VdcStorageProfile, error) { @@ -1190,6 +1175,38 @@ func QueryAdminOrgVdcStorageProfileByID(vcdCli *VCDClient, id string) (*types.Qu return results.Results.AdminOrgVdcStorageProfileRecord[0], nil } +// queryAdminOrgVdcStorageProfilesByVdcId finds all Storage Profiles of a VDC +func queryAdminOrgVdcStorageProfilesByVdcId(client *Client, vdcId string) ([]*types.QueryResultAdminOrgVdcStorageProfileRecordType, error) { + if !client.IsSysAdmin { + return nil, errors.New("can't query type QueryResultAdminOrgVdcStorageProfileRecordType as Tenant user") + } + results, err := client.QueryWithNotEncodedParams(nil, map[string]string{ + "type": types.QtAdminOrgVdcStorageProfile, + "filter": fmt.Sprintf("vdc==%s", url.QueryEscape(vdcId)), + "filterEncoded": "true", + }) + if err != nil { + return nil, err + } + return results.Results.AdminOrgVdcStorageProfileRecord, nil +} + +// queryOrgVdcStorageProfilesByVdcId finds all Storage Profiles of a VDC +func queryOrgVdcStorageProfilesByVdcId(client *Client, vdcId string) ([]*types.QueryResultOrgVdcStorageProfileRecordType, error) { + if client.IsSysAdmin { + return nil, errors.New("can't query type QueryResultOrgVdcStorageProfileRecordType as System administrator") + } + results, err := client.QueryWithNotEncodedParams(nil, map[string]string{ + "type": types.QtOrgVdcStorageProfile, + "filter": fmt.Sprintf("vdc==%s", url.QueryEscape(vdcId)), + "filterEncoded": "true", + }) + if err != nil { + return nil, err + } + return results.Results.OrgVdcStorageProfileRecord, nil +} + // QueryOrgVdcStorageProfileByID finds a StorageProfile of VDC by ID func QueryOrgVdcStorageProfileByID(vcdCli *VCDClient, id string) (*types.QueryResultOrgVdcStorageProfileRecordType, error) { if vcdCli.Client.IsSysAdmin { From ff26daa6a21c57b03a55490aa5aa3aaf00bdf1b1 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 6 Feb 2024 11:59:50 +0100 Subject: [PATCH 019/115] Refactoring Signed-off-by: abarreiro --- govcd/cse.go | 32 ++++--- govcd/cse_template.go | 150 ++++++++++++++++-------------- govcd/cse_type.go | 18 +++- govcd/cse_util.go | 207 ++++++++++++++++++++++-------------------- govcd/cse_yaml.go | 5 +- 5 files changed, 228 insertions(+), 184 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 27ee3bf90..179966e45 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -13,8 +13,8 @@ import ( // timeout is 0, it waits forever for the cluster creation. // // If the timeout is reached and the cluster is not available (in "provisioned" state), it will return a non-nil CseKubernetesCluster -// with only the cluster ID and an error. This means that the cluster will be left in VCD in any state, and it can be retrieved with -// Org.CseGetKubernetesClusterById manually. +// with only the cluster ID and an error. This means that the cluster will be left in VCD in any state, and it can be retrieved afterward +// with Org.CseGetKubernetesClusterById and the returned ID. // // If the cluster is created correctly, returns all the available data in CseKubernetesCluster or an error if some of the fields // of the created cluster cannot be calculated or retrieved. @@ -37,15 +37,15 @@ func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterSettings, timeo // the created cluster. One can manually check the status of the cluster with Org.CseGetKubernetesClusterById and the result of this method. func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) (string, error) { if org == nil { - return "", fmt.Errorf("receiver Organization is nil") + return "", fmt.Errorf("CseCreateKubernetesClusterAsync cannot be called on a nil Organization receiver") } - goTemplateContents, err := cseClusterSettingsToInternal(clusterData, *org) + internalSettings, err := clusterData.toCseClusterSettingsInternal(*org) if err != nil { - return "", err + return "", fmt.Errorf("error creating the CSE Kubernetes cluster: %s", err) } - rdeContents, err := getCseKubernetesClusterCreationPayload(goTemplateContents) + rdeContents, err := internalSettings.getKubernetesClusterCreationPayload() if err != nil { return "", err } @@ -55,14 +55,15 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) return "", err } - rde, err := createRdeAndPoll(org.client, "vmware", "capvcdCluster", cseSubcomponents.CapvcdRdeTypeVersion, types.DefinedEntity{ - EntityType: goTemplateContents.RdeType.ID, - Name: goTemplateContents.Name, - Entity: rdeContents, - }, &TenantContext{ - OrgId: org.Org.ID, - OrgName: org.Org.Name, - }) + rde, err := createRdeAndPoll(org.client, "vmware", "capvcdCluster", cseSubcomponents.CapvcdRdeTypeVersion, + types.DefinedEntity{ + EntityType: internalSettings.RdeType.ID, + Name: internalSettings.Name, + Entity: rdeContents, + }, &TenantContext{ + OrgId: org.Org.ID, + OrgName: org.Org.Name, + }) if err != nil { return "", err } @@ -105,7 +106,8 @@ func (cluster *CseKubernetesCluster) GetKubeconfig() (string, error) { return "", err } - // Auxiliary wrapper of the result, as the invocation returns the full RDE. + // Auxiliary wrapper of the result, as the invocation returns the RDE and + // what we need is inside of it. type invocationResult struct { Capvcd types.Capvcd `json:"entity,omitempty"` } diff --git a/govcd/cse_template.go b/govcd/cse_template.go index bdc92b533..f1cf7030a 100644 --- a/govcd/cse_template.go +++ b/govcd/cse_template.go @@ -6,46 +6,48 @@ import ( "encoding/base64" "encoding/json" "fmt" - semver "github.com/hashicorp/go-version" "strconv" "strings" "text/template" ) -// getCseKubernetesClusterCreationPayload gets the payload for the RDE that will trigger a Kubernetes cluster creation. +// getKubernetesClusterCreationPayload gets the payload for the RDE that will trigger a Kubernetes cluster creation. // It generates a valid YAML that is embedded inside the RDE JSON, then it is returned as an unmarshaled // generic map, that allows to be sent to VCD as it is. -func getCseKubernetesClusterCreationPayload(goTemplateContents cseClusterSettingsInternal) (map[string]interface{}, error) { - capiYaml, err := generateCapiYaml(goTemplateContents) +func (clusterSettings *cseClusterSettingsInternal) getKubernetesClusterCreationPayload() (map[string]interface{}, error) { + if clusterSettings == nil { + return nil, fmt.Errorf("the receiver cluster settings is nil") + } + capiYaml, err := clusterSettings.generateCapiYaml() if err != nil { return nil, err } args := map[string]string{ - "Name": goTemplateContents.Name, - "Org": goTemplateContents.OrganizationName, - "VcdUrl": goTemplateContents.VcdUrl, - "Vdc": goTemplateContents.VdcName, + "Name": clusterSettings.Name, + "Org": clusterSettings.OrganizationName, + "VcdUrl": clusterSettings.VcdUrl, + "Vdc": clusterSettings.VdcName, "Delete": "false", "ForceDelete": "false", - "AutoRepairOnErrors": strconv.FormatBool(goTemplateContents.AutoRepairOnErrors), - "ApiToken": goTemplateContents.ApiToken, + "AutoRepairOnErrors": strconv.FormatBool(clusterSettings.AutoRepairOnErrors), + "ApiToken": clusterSettings.ApiToken, "CapiYaml": capiYaml, } - if goTemplateContents.DefaultStorageClass.StorageProfileName != "" { - args["DefaultStorageClassStorageProfile"] = goTemplateContents.DefaultStorageClass.StorageProfileName - args["DefaultStorageClassName"] = goTemplateContents.DefaultStorageClass.Name - args["DefaultStorageClassUseDeleteReclaimPolicy"] = strconv.FormatBool(goTemplateContents.DefaultStorageClass.UseDeleteReclaimPolicy) - args["DefaultStorageClassFileSystem"] = goTemplateContents.DefaultStorageClass.Filesystem + if clusterSettings.DefaultStorageClass.StorageProfileName != "" { + args["DefaultStorageClassStorageProfile"] = clusterSettings.DefaultStorageClass.StorageProfileName + args["DefaultStorageClassName"] = clusterSettings.DefaultStorageClass.Name + args["DefaultStorageClassUseDeleteReclaimPolicy"] = strconv.FormatBool(clusterSettings.DefaultStorageClass.UseDeleteReclaimPolicy) + args["DefaultStorageClassFileSystem"] = clusterSettings.DefaultStorageClass.Filesystem } - rdeTmpl, err := getCseTemplate(goTemplateContents.CseVersion, "rde") + rdeTmpl, err := getCseTemplate(clusterSettings.CseVersion, "rde") if err != nil { return nil, err } - capvcdEmpty := template.Must(template.New(goTemplateContents.Name).Parse(rdeTmpl)) + capvcdEmpty := template.Must(template.New(clusterSettings.Name).Parse(rdeTmpl)) buf := &bytes.Buffer{} if err := capvcdEmpty.Execute(buf, args); err != nil { return nil, fmt.Errorf("could not render the Go template with the CAPVCD JSON: %s", err) @@ -61,18 +63,22 @@ func getCseKubernetesClusterCreationPayload(goTemplateContents cseClusterSetting } // generateNodePoolYaml generates YAML blocks corresponding to the Kubernetes node pools. -func generateNodePoolYaml(clusterDetails cseClusterSettingsInternal) (string, error) { - workerPoolTmpl, err := getCseTemplate(clusterDetails.CseVersion, "capiyaml_workerpool") +func (clusterSettings *cseClusterSettingsInternal) generateNodePoolYaml() (string, error) { + if clusterSettings == nil { + return "", fmt.Errorf("the receiver cluster settings is nil") + } + + workerPoolTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_workerpool") if err != nil { return "", err } - nodePoolEmptyTmpl := template.Must(template.New(clusterDetails.Name + "-worker-pool").Parse(workerPoolTmpl)) + nodePoolEmptyTmpl := template.Must(template.New(clusterSettings.Name + "-worker-pool").Parse(workerPoolTmpl)) resultYaml := "" buf := &bytes.Buffer{} // We can have many worker pools, we build a YAML object for each one of them. - for _, workerPool := range clusterDetails.WorkerPools { + for _, workerPool := range clusterSettings.WorkerPools { // Check the correctness of the compute policies in the node pool block if workerPool.PlacementPolicyName != "" && workerPool.VGpuPolicyName != "" { @@ -84,18 +90,18 @@ func generateNodePoolYaml(clusterDetails cseClusterSettingsInternal) (string, er } if err := nodePoolEmptyTmpl.Execute(buf, map[string]string{ - "ClusterName": clusterDetails.Name, + "ClusterName": clusterSettings.Name, "NodePoolName": workerPool.Name, - "TargetNamespace": clusterDetails.Name + "-ns", - "Catalog": clusterDetails.CatalogName, - "VAppTemplate": clusterDetails.KubernetesTemplateOvaName, + "TargetNamespace": clusterSettings.Name + "-ns", + "Catalog": clusterSettings.CatalogName, + "VAppTemplate": clusterSettings.KubernetesTemplateOvaName, "NodePoolSizingPolicy": workerPool.SizingPolicyName, "NodePoolPlacementPolicy": placementPolicy, // Can be either Placement or vGPU policy "NodePoolStorageProfile": workerPool.StorageProfileName, "NodePoolDiskSize": fmt.Sprintf("%dGi", workerPool.DiskSizeGi), "NodePoolEnableGpu": strconv.FormatBool(workerPool.VGpuPolicyName != ""), "NodePoolMachineCount": strconv.Itoa(workerPool.MachineCount), - "KubernetesVersion": clusterDetails.TkgVersionBundle.KubernetesVersion, + "KubernetesVersion": clusterSettings.TkgVersionBundle.KubernetesVersion, }); err != nil { return "", fmt.Errorf("could not generate a correct Node Pool YAML: %s", err) } @@ -106,27 +112,31 @@ func generateNodePoolYaml(clusterDetails cseClusterSettingsInternal) (string, er } // generateMemoryHealthCheckYaml generates a YAML block corresponding to the Kubernetes memory health check. -func generateMemoryHealthCheckYaml(vcdKeConfig vcdKeConfig, cseVersion semver.Version, clusterName string) (string, error) { - if vcdKeConfig.NodeStartupTimeout == "" && vcdKeConfig.NodeUnknownTimeout == "" && vcdKeConfig.NodeNotReadyTimeout == "" && - vcdKeConfig.MaxUnhealthyNodesPercentage == 0 { +func (clusterSettings *cseClusterSettingsInternal) generateMemoryHealthCheckYaml() (string, error) { + if clusterSettings == nil { + return "", fmt.Errorf("the receiver cluster settings is nil") + } + + if clusterSettings.VcdKeConfig.NodeStartupTimeout == "" && clusterSettings.VcdKeConfig.NodeUnknownTimeout == "" && clusterSettings.VcdKeConfig.NodeNotReadyTimeout == "" && + clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage == 0 { return "", nil } - mhcTmpl, err := getCseTemplate(cseVersion, "capiyaml_mhc") + mhcTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_mhc") if err != nil { return "", err } - mhcEmptyTmpl := template.Must(template.New(clusterName + "-mhc").Parse(mhcTmpl)) + mhcEmptyTmpl := template.Must(template.New(clusterSettings.Name + "-mhc").Parse(mhcTmpl)) buf := &bytes.Buffer{} if err := mhcEmptyTmpl.Execute(buf, map[string]string{ - "ClusterName": clusterName, - "TargetNamespace": clusterName + "-ns", - "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", vcdKeConfig.MaxUnhealthyNodesPercentage), // With the 'percentage' suffix - "NodeStartupTimeout": fmt.Sprintf("%ss", vcdKeConfig.NodeStartupTimeout), // With the 'second' suffix - "NodeUnknownTimeout": fmt.Sprintf("%ss", vcdKeConfig.NodeUnknownTimeout), // With the 'second' suffix - "NodeNotReadyTimeout": fmt.Sprintf("%ss", vcdKeConfig.NodeNotReadyTimeout), // With the 'second' suffix + "ClusterName": clusterSettings.Name, + "TargetNamespace": clusterSettings.Name + "-ns", + "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage), // With the 'percentage' suffix + "NodeStartupTimeout": fmt.Sprintf("%ss", clusterSettings.VcdKeConfig.NodeStartupTimeout), // With the 'second' suffix + "NodeUnknownTimeout": fmt.Sprintf("%ss", clusterSettings.VcdKeConfig.NodeUnknownTimeout), // With the 'second' suffix + "NodeNotReadyTimeout": fmt.Sprintf("%ss", clusterSettings.VcdKeConfig.NodeNotReadyTimeout), // With the 'second' suffix }); err != nil { return "", fmt.Errorf("could not generate a correct Memory Health Check YAML: %s", err) } @@ -137,53 +147,57 @@ func generateMemoryHealthCheckYaml(vcdKeConfig vcdKeConfig, cseVersion semver.Ve // generateCapiYaml generates the YAML string that is required during Kubernetes cluster creation, to be embedded // in the CAPVCD cluster JSON payload. This function picks data from the Terraform schema and the createClusterDto to // populate several Go templates and build a final YAML. -func generateCapiYaml(clusterDetails cseClusterSettingsInternal) (string, error) { - clusterTmpl, err := getCseTemplate(clusterDetails.CseVersion, "capiyaml_cluster") +func (clusterSettings *cseClusterSettingsInternal) generateCapiYaml() (string, error) { + if clusterSettings == nil { + return "", fmt.Errorf("the receiver cluster settings is nil") + } + + clusterTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_cluster") if err != nil { return "", err } // This YAML snippet contains special strings, such as "%,", that render wrong using the Go template engine sanitizedTemplate := strings.NewReplacer("%", "%%").Replace(clusterTmpl) - capiYamlEmpty := template.Must(template.New(clusterDetails.Name + "-cluster").Parse(sanitizedTemplate)) + capiYamlEmpty := template.Must(template.New(clusterSettings.Name + "-cluster").Parse(sanitizedTemplate)) - nodePoolYaml, err := generateNodePoolYaml(clusterDetails) + nodePoolYaml, err := clusterSettings.generateNodePoolYaml() if err != nil { return "", err } - memoryHealthCheckYaml, err := generateMemoryHealthCheckYaml(clusterDetails.VcdKeConfig, clusterDetails.CseVersion, clusterDetails.Name) + memoryHealthCheckYaml, err := clusterSettings.generateMemoryHealthCheckYaml() if err != nil { return "", err } args := map[string]string{ - "ClusterName": clusterDetails.Name, - "TargetNamespace": clusterDetails.Name + "-ns", - "TkrVersion": clusterDetails.TkgVersionBundle.TkrVersion, - "TkgVersion": clusterDetails.TkgVersionBundle.TkgVersion, - "UsernameB64": base64.StdEncoding.EncodeToString([]byte(clusterDetails.Owner)), - "ApiTokenB64": base64.StdEncoding.EncodeToString([]byte(clusterDetails.ApiToken)), - "PodCidr": clusterDetails.PodCidr, - "ServiceCidr": clusterDetails.ServiceCidr, - "VcdSite": clusterDetails.VcdUrl, - "Org": clusterDetails.OrganizationName, - "OrgVdc": clusterDetails.VdcName, - "OrgVdcNetwork": clusterDetails.NetworkName, - "Catalog": clusterDetails.CatalogName, - "VAppTemplate": clusterDetails.KubernetesTemplateOvaName, - "ControlPlaneSizingPolicy": clusterDetails.ControlPlane.SizingPolicyName, - "ControlPlanePlacementPolicy": clusterDetails.ControlPlane.PlacementPolicyName, - "ControlPlaneStorageProfile": clusterDetails.ControlPlane.StorageProfileName, - "ControlPlaneDiskSize": fmt.Sprintf("%dGi", clusterDetails.ControlPlane.DiskSizeGi), - "ControlPlaneMachineCount": strconv.Itoa(clusterDetails.ControlPlane.MachineCount), - "ControlPlaneEndpoint": clusterDetails.ControlPlane.Ip, - "DnsVersion": clusterDetails.TkgVersionBundle.CoreDnsVersion, - "EtcdVersion": clusterDetails.TkgVersionBundle.EtcdVersion, - "ContainerRegistryUrl": clusterDetails.VcdKeConfig.ContainerRegistryUrl, - "KubernetesVersion": clusterDetails.TkgVersionBundle.KubernetesVersion, - "SshPublicKey": clusterDetails.SshPublicKey, - "VirtualIpSubnet": clusterDetails.VirtualIpSubnet, + "ClusterName": clusterSettings.Name, + "TargetNamespace": clusterSettings.Name + "-ns", + "TkrVersion": clusterSettings.TkgVersionBundle.TkrVersion, + "TkgVersion": clusterSettings.TkgVersionBundle.TkgVersion, + "UsernameB64": base64.StdEncoding.EncodeToString([]byte(clusterSettings.Owner)), + "ApiTokenB64": base64.StdEncoding.EncodeToString([]byte(clusterSettings.ApiToken)), + "PodCidr": clusterSettings.PodCidr, + "ServiceCidr": clusterSettings.ServiceCidr, + "VcdSite": clusterSettings.VcdUrl, + "Org": clusterSettings.OrganizationName, + "OrgVdc": clusterSettings.VdcName, + "OrgVdcNetwork": clusterSettings.NetworkName, + "Catalog": clusterSettings.CatalogName, + "VAppTemplate": clusterSettings.KubernetesTemplateOvaName, + "ControlPlaneSizingPolicy": clusterSettings.ControlPlane.SizingPolicyName, + "ControlPlanePlacementPolicy": clusterSettings.ControlPlane.PlacementPolicyName, + "ControlPlaneStorageProfile": clusterSettings.ControlPlane.StorageProfileName, + "ControlPlaneDiskSize": fmt.Sprintf("%dGi", clusterSettings.ControlPlane.DiskSizeGi), + "ControlPlaneMachineCount": strconv.Itoa(clusterSettings.ControlPlane.MachineCount), + "ControlPlaneEndpoint": clusterSettings.ControlPlane.Ip, + "DnsVersion": clusterSettings.TkgVersionBundle.CoreDnsVersion, + "EtcdVersion": clusterSettings.TkgVersionBundle.EtcdVersion, + "ContainerRegistryUrl": clusterSettings.VcdKeConfig.ContainerRegistryUrl, + "KubernetesVersion": clusterSettings.TkgVersionBundle.KubernetesVersion, + "SshPublicKey": clusterSettings.SshPublicKey, + "VirtualIpSubnet": clusterSettings.VirtualIpSubnet, } buf := &bytes.Buffer{} diff --git a/govcd/cse_type.go b/govcd/cse_type.go index b634e4efd..ebd91e228 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -7,7 +7,7 @@ import ( "time" ) -// CseKubernetesCluster is a type for handling a Kubernetes cluster created by the Container Service Extension (CSE) +// CseKubernetesCluster is a type for managing an existing Kubernetes cluster created by the Container Service Extension (CSE) type CseKubernetesCluster struct { CseClusterSettings ID string @@ -79,8 +79,8 @@ type CseWorkerPoolSettings struct { type CseDefaultStorageClassSettings struct { StorageProfileId string Name string - ReclaimPolicy string - Filesystem string + ReclaimPolicy string // Must be either "delete" or "retain" + Filesystem string // Must be either "ext4" or "xfs" } // CseClusterUpdateInput defines the required configuration that a Container Service Extension (CSE) Kubernetes cluster needs in order to be updated. @@ -141,6 +141,18 @@ type cseClusterSettingsInternal struct { AutoRepairOnErrors bool } +// tkgVersionBundle is a type that contains all the versions of the components of +// a Kubernetes cluster that can be obtained with the Kubernetes Template OVA name, downloaded +// from VMware Customer connect: +// https://customerconnect.vmware.com/downloads/details?downloadGroup=TKG-240&productId=1400 +type tkgVersionBundle struct { + EtcdVersion string + CoreDnsVersion string + TkgVersion string + TkrVersion string + KubernetesVersion string +} + // cseControlPlaneSettingsInternal defines the Control Plane inside cseClusterSettingsInternal type cseControlPlaneSettingsInternal struct { MachineCount int diff --git a/govcd/cse_util.go b/govcd/cse_util.go index f8da5ab0b..509426725 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -14,19 +14,19 @@ import ( "time" ) -// getCseComponentsVersions gets the CSE components versions from its version. +// getCseComponentsVersions gets the versions of the sub-components that are part of Container Service Extension. // TODO: Is this really necessary? What happens in UI if I have a 1.1.0-1.2.0-1.0.0 (4.2) cluster and then CSE is updated to 4.3? func getCseComponentsVersions(cseVersion semver.Version) (*cseComponentsVersions, error) { - v42, _ := semver.NewVersion("4.1") + v41, _ := semver.NewVersion("4.1") - if cseVersion.Equal(v42) { + if cseVersion.Equal(v41) { return &cseComponentsVersions{ VcdKeConfigRdeTypeVersion: "1.1.0", CapvcdRdeTypeVersion: "1.2.0", CseInterfaceVersion: "1.0.0", }, nil } - return nil, fmt.Errorf("not supported version %s", cseVersion.String()) + return nil, fmt.Errorf("the Container Service Extension version '%s' is not supported", cseVersion.String()) } // cseConvertToCseKubernetesClusterType takes a generic RDE that must represent an existing CSE Kubernetes cluster, @@ -117,9 +117,10 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu if len(result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs) == 0 { return nil, fmt.Errorf("could not read VDCs from Capvcd type") } + result.VdcId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Id + // FIXME: This is a workaround, because for some reason the ID contains the VDC name instead of the VDC ID. // Once this is fixed, this conditional should not be needed anymore. - result.VdcId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Id if result.VdcId == result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Name { vdcs, err := queryOrgVdcList(rde.client, map[string]string{}) if err != nil { @@ -205,6 +206,9 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu } } + // NOTE: We get the remaining elements from the CAPI YAML, despite they are also inside capvcdType.Status. + // The reason is that any change on the cluster is immediately reflected in the CAPI YAML, but not in the capvcdType.Status + // elements, which may take 10 minutes to be refreshed. yamlDocuments, err := unmarshalMultipleYamlDocuments(result.capvcdType.Spec.CapiYaml) if err != nil { return nil, err @@ -212,7 +216,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu // We need a map of worker pools and not a slice, because there are two types of YAML documents // that contain data about a specific worker pool (VCDMachineTemplate and MachineDeployment), and we can get them in no - // particular order, so we store the worker pools with their name as key. This way we can easily fetch them and override them. + // particular order, so we store the worker pools with their name as key. This way we can easily (O(1)) fetch and update them. workerPools := map[string]CseWorkerPoolSettings{} for _, yamlDocument := range yamlDocuments { switch yamlDocument["kind"] { @@ -419,62 +423,67 @@ func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinu return fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, capvcd.Status.VcdKe.State) } -// validate validates the CSE Kubernetes cluster creation input data. Returns an error if some of the fields is wrong. +// validate validates the receiver CseClusterSettings. Returns an error if any of the fields is empty or wrong. func (input *CseClusterSettings) validate() error { + if input == nil { + return fmt.Errorf("the receiver CseClusterSettings cannot be nil") + } + // This regular expression is used to validate the constraints placed by Container Service Extension on the names + // of the components of the Kubernetes clusters: + // Names must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters. cseNamesRegex, err := regexp.Compile(`^[a-z](?:[a-z0-9-]{0,29}[a-z0-9])?$`) if err != nil { return fmt.Errorf("could not compile regular expression '%s'", err) } + _, err = getCseComponentsVersions(input.CseVersion) + if err != nil { + return err + } if !cseNamesRegex.MatchString(input.Name) { - return fmt.Errorf("the cluster name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", input.Name) + return fmt.Errorf("the name '%s' must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters", input.Name) } - if input.OrganizationId == "" { return fmt.Errorf("the Organization ID is required") } if input.VdcId == "" { return fmt.Errorf("the VDC ID is required") } - if input.KubernetesTemplateOvaId == "" { - return fmt.Errorf("the Kubernetes template OVA ID is required") - } if input.NetworkId == "" { return fmt.Errorf("the Network ID is required") } - _, err = getCseComponentsVersions(input.CseVersion) - if err != nil { - return fmt.Errorf("the CSE version '%s' is not supported", input.CseVersion.String()) + if input.KubernetesTemplateOvaId == "" { + return fmt.Errorf("the Kubernetes Template OVA ID is required") } if input.ControlPlane.MachineCount < 1 || input.ControlPlane.MachineCount%2 == 0 { - return fmt.Errorf("number of control plane nodes must be odd and higher than 0, but it was '%d'", input.ControlPlane.MachineCount) + return fmt.Errorf("number of Control Plane nodes must be odd and higher than 0, but it was '%d'", input.ControlPlane.MachineCount) } if input.ControlPlane.DiskSizeGi < 20 { return fmt.Errorf("disk size for the Control Plane in Gibibytes (Gi) must be at least 20, but it was '%d'", input.ControlPlane.DiskSizeGi) } if len(input.WorkerPools) == 0 { - return fmt.Errorf("there must be at least one Worker pool") + return fmt.Errorf("there must be at least one Worker Pool") } for _, workerPool := range input.WorkerPools { - if !cseNamesRegex.MatchString(workerPool.Name) { - return fmt.Errorf("the Worker pool name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", workerPool.Name) + if workerPool.MachineCount < 1 { + return fmt.Errorf("number of Worker Pool '%s' nodes must higher than 0, but it was '%d'", workerPool.Name, workerPool.MachineCount) } if workerPool.DiskSizeGi < 20 { - return fmt.Errorf("disk size for the Worker pool '%s' in Gibibytes (Gi) must be at least 20, but it was '%d'", workerPool.Name, workerPool.DiskSizeGi) + return fmt.Errorf("disk size for the Worker Pool '%s' in Gibibytes (Gi) must be at least 20, but it was '%d'", workerPool.Name, workerPool.DiskSizeGi) } - if workerPool.MachineCount < 1 { - return fmt.Errorf("number of Worker pool '%s' nodes must higher than 0, but it was '%d'", workerPool.Name, workerPool.MachineCount) + if !cseNamesRegex.MatchString(workerPool.Name) { + return fmt.Errorf("the Worker Pool name '%s' must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters", workerPool.Name) } } - if input.DefaultStorageClass != nil { + if input.DefaultStorageClass != nil { // This field is optional if !cseNamesRegex.MatchString(input.DefaultStorageClass.Name) { - return fmt.Errorf("the Default Storage Class name is required and must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters, but it was: '%s'", input.DefaultStorageClass.Name) + return fmt.Errorf("the Default Storage Class name '%s' must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters", input.DefaultStorageClass.Name) } if input.DefaultStorageClass.StorageProfileId == "" { return fmt.Errorf("the Storage Profile ID for the Default Storage Class is required") } if input.DefaultStorageClass.ReclaimPolicy != "delete" && input.DefaultStorageClass.ReclaimPolicy != "retain" { - return fmt.Errorf("the reclaim policy for the Default Storage Class must be either 'delete' or 'retain', but it was '%s'", input.DefaultStorageClass.ReclaimPolicy) + return fmt.Errorf("the Reclaim Policy for the Default Storage Class must be either 'delete' or 'retain', but it was '%s'", input.DefaultStorageClass.ReclaimPolicy) } if input.DefaultStorageClass.Filesystem != "ext4" && input.DefaultStorageClass.ReclaimPolicy != "xfs" { return fmt.Errorf("the filesystem for the Default Storage Class must be either 'ext4' or 'xfs', but it was '%s'", input.DefaultStorageClass.Filesystem) @@ -489,70 +498,74 @@ func (input *CseClusterSettings) validate() error { if input.ServiceCidr == "" { return fmt.Errorf("the Service CIDR is required") } - return nil } -// cseClusterSettingsToInternal transforms user input data (CseClusterSettings) into the final payload that -// will be used to render the Go templates that define a Kubernetes cluster creation payload (cseClusterSettingsInternal). -func cseClusterSettingsToInternal(input CseClusterSettings, org Org) (cseClusterSettingsInternal, error) { - output := cseClusterSettingsInternal{} +// toCseClusterSettingsInternal transforms user input data (CseClusterSettings) into the final payload that +// will be used to define a Container Service Extension Kubernetes cluster (cseClusterSettingsInternal). +// +// For example, the most relevant transformation is the change of the item IDs that are present in CseClusterSettings +// (such as CseClusterSettings.KubernetesTemplateOvaId) to their corresponding Names (e.g. cseClusterSettingsInternal.KubernetesTemplateOvaName), +// which are the identifiers that Container Service Extension uses internally. +func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClusterSettingsInternal, error) { err := input.validate() if err != nil { - return output, err + return nil, err } + output := &cseClusterSettingsInternal{} if org.Org == nil { - return output, fmt.Errorf("the Organization is nil") + return nil, fmt.Errorf("could not retrieve the Organization, it is nil") } - output.OrganizationName = org.Org.Name vdc, err := org.GetVDCById(input.VdcId, true) if err != nil { - return output, fmt.Errorf("could not retrieve the VDC with ID '%s': %s", input.VdcId, err) + return nil, fmt.Errorf("could not retrieve the VDC with ID '%s': %s", input.VdcId, err) } output.VdcName = vdc.Vdc.Name vAppTemplate, err := getVAppTemplateById(org.client, input.KubernetesTemplateOvaId) if err != nil { - return output, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", input.KubernetesTemplateOvaId, err) + return nil, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", input.KubernetesTemplateOvaId, err) } output.KubernetesTemplateOvaName = vAppTemplate.VAppTemplate.Name tkgVersions, err := getTkgVersionBundleFromVAppTemplateName(vAppTemplate.VAppTemplate.Name) if err != nil { - return output, fmt.Errorf("could not retrieve the required information from the Kubernetes Template OVA: %s", err) + return nil, fmt.Errorf("could not retrieve the required information from the Kubernetes Template OVA: %s", err) } output.TkgVersionBundle = tkgVersions catalogName, err := vAppTemplate.GetCatalogName() if err != nil { - return output, fmt.Errorf("could not retrieve the Catalog name where the the Kubernetes Template OVA '%s' is hosted: %s", input.KubernetesTemplateOvaId, err) + return nil, fmt.Errorf("could not retrieve the Catalog name where the the Kubernetes Template OVA '%s' (%s) is hosted: %s", input.KubernetesTemplateOvaId, vAppTemplate.VAppTemplate.Name, err) } output.CatalogName = catalogName network, err := vdc.GetOrgVdcNetworkById(input.NetworkId, true) if err != nil { - return output, fmt.Errorf("could not retrieve the Org VDC Network with ID '%s': %s", input.NetworkId, err) + return nil, fmt.Errorf("could not retrieve the Org VDC Network with ID '%s': %s", input.NetworkId, err) } output.NetworkName = network.OrgVDCNetwork.Name - currentCseVersion, err := getCseComponentsVersions(input.CseVersion) + cseComponentsVersions, err := getCseComponentsVersions(input.CseVersion) if err != nil { - return output, fmt.Errorf("the CSE version '%s' is not supported: %s", input.CseVersion.String(), err) + return nil, err } - rdeType, err := getRdeType(org.client, "vmware", "capvcdCluster", currentCseVersion.CapvcdRdeTypeVersion) + rdeType, err := getRdeType(org.client, "vmware", "capvcdCluster", cseComponentsVersions.CapvcdRdeTypeVersion) if err != nil { - return output, err + return nil, err } output.RdeType = rdeType.DefinedEntityType - // The input to create a cluster uses different entities IDs, but CSE cluster creation process uses names. - // For that reason, we need to transform IDs to Names by querying VCD. This process is optimized with a tiny "nameToIdCache" map. + // The input to create a cluster uses different entities IDs, but CSE cluster creation process uses Names. + // For that reason, we need to transform IDs to Names by querying VCD. This process is optimized with a tiny cache map. idToNameCache := map[string]string{ - "": "", // Default empty value to map optional values that were not set + "": "", // Default empty value to map optional values that were not set, to avoid extra checks. For example, an empty vGPU Policy. } + + // Gather all the IDs of the Compute Policies and Storage Profiles, so we can transform them to Names in bulk. var computePolicyIds []string var storageProfileIds []string for _, w := range input.WorkerPools { @@ -562,11 +575,13 @@ func cseClusterSettingsToInternal(input CseClusterSettings, org Org) (cseCluster computePolicyIds = append(computePolicyIds, input.ControlPlane.SizingPolicyId, input.ControlPlane.PlacementPolicyId) storageProfileIds = append(storageProfileIds, input.ControlPlane.StorageProfileId, input.DefaultStorageClass.StorageProfileId) + // Retrieve the Compute Policies and Storage Profiles names and put them in the cache. The cache + // reduces the calls to VCD. The URN format used by VCD guarantees that IDs are unique, so there is no possibility of clashes here. for _, id := range storageProfileIds { if _, alreadyPresent := idToNameCache[id]; !alreadyPresent { storageProfile, err := getStorageProfileById(org.client, id) if err != nil { - return output, fmt.Errorf("could not retrieve Storage Profile with ID '%s': %s", id, err) + return nil, fmt.Errorf("could not retrieve Storage Profile with ID '%s': %s", id, err) } idToNameCache[id] = storageProfile.Name } @@ -575,13 +590,13 @@ func cseClusterSettingsToInternal(input CseClusterSettings, org Org) (cseCluster if _, alreadyPresent := idToNameCache[id]; !alreadyPresent { computePolicy, err := getVdcComputePolicyV2ById(org.client, id) if err != nil { - return output, fmt.Errorf("could not retrieve Compute Policy with ID '%s': %s", id, err) + return nil, fmt.Errorf("could not retrieve Compute Policy with ID '%s': %s", id, err) } idToNameCache[id] = computePolicy.VdcComputePolicyV2.Name } } - // Now that everything is cached in memory, we can build the Node pools and Storage Class payloads + // Now that everything is cached in memory, we can build the Node pools and Storage Class payloads in a trivial way. output.WorkerPools = make([]cseWorkerPoolSettingsInternal, len(input.WorkerPools)) for i, w := range input.WorkerPools { output.WorkerPools[i] = cseWorkerPoolSettingsInternal{ @@ -615,25 +630,23 @@ func cseClusterSettingsToInternal(input CseClusterSettings, org Org) (cseCluster } } - cseVersions, err := getCseComponentsVersions(input.CseVersion) + vcdKeConfig, err := getVcdKeConfig(org.client, cseComponentsVersions.VcdKeConfigRdeTypeVersion, input.NodeHealthCheck) if err != nil { - return output, err + return nil, err } - - vcdKeConfig, err := getVcdKeConfig(org.client, cseVersions.VcdKeConfigRdeTypeVersion, input.NodeHealthCheck) - if err != nil { - return output, err + if vcdKeConfig != nil { + output.VcdKeConfig = *vcdKeConfig } - output.VcdKeConfig = vcdKeConfig output.Owner = input.Owner if input.Owner == "" { sessionInfo, err := org.client.GetSessionInfo() if err != nil { - return output, fmt.Errorf("error getting the owner of the cluster: %s", err) + return nil, fmt.Errorf("error getting the Owner: %s", err) } output.Owner = sessionInfo.User.Name } + output.VcdUrl = strings.Replace(org.client.VCDHREF.String(), "/api", "", 1) // These don't change, don't need mapping @@ -649,52 +662,42 @@ func cseClusterSettingsToInternal(input CseClusterSettings, org Org) (cseCluster return output, nil } -// tkgVersionBundle is a type that contains all the versions of the components of -// a Kubernetes cluster that can be obtained with the vApp Template name, downloaded -// from VMware Customer connect: -// https://customerconnect.vmware.com/downloads/details?downloadGroup=TKG-240&productId=1400 -type tkgVersionBundle struct { - EtcdVersion string - CoreDnsVersion string - TkgVersion string - TkrVersion string - KubernetesVersion string -} - // getTkgVersionBundleFromVAppTemplateName returns a tkgVersionBundle with the details of -// all the Kubernetes cluster components versions given a valid vApp Template name, that should -// correspond to a Kubernetes template. If it is not a valid vApp Template, returns an error. -func getTkgVersionBundleFromVAppTemplateName(ovaName string) (tkgVersionBundle, error) { +// all the Kubernetes cluster components versions given a valid Kubernetes Template OVA name. +// If it is not a valid Kubernetes Template OVA, returns an error. +func getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName string) (tkgVersionBundle, error) { result := tkgVersionBundle{} - if strings.Contains(ovaName, "photon") { - return result, fmt.Errorf("the OVA '%s' uses Photon, and it is not supported", ovaName) + if strings.Contains(kubernetesTemplateOvaName, "photon") { + return result, fmt.Errorf("the Kubernetes Template OVA '%s' uses Photon, and it is not supported", kubernetesTemplateOvaName) } - cutPosition := strings.LastIndex(ovaName, "kube-") + cutPosition := strings.LastIndex(kubernetesTemplateOvaName, "kube-") if cutPosition < 0 { - return result, fmt.Errorf("the OVA '%s' is not a Kubernetes template OVA", ovaName) + return result, fmt.Errorf("the OVA '%s' is not a Kubernetes template OVA", kubernetesTemplateOvaName) } - parsedOvaName := strings.ReplaceAll(ovaName, ".ova", "")[cutPosition+len("kube-"):] + parsedOvaName := strings.ReplaceAll(kubernetesTemplateOvaName, ".ova", "")[cutPosition+len("kube-"):] - cseTkgVersionsJson, err := cseFiles.ReadFile("cse/tkg_versions.json") + tkgVersionsMap := "cse/tkg_versions.json" + cseTkgVersionsJson, err := cseFiles.ReadFile(tkgVersionsMap) if err != nil { - return result, err + return result, fmt.Errorf("failed reading %s: %s", tkgVersionsMap, err) } versionsMap := map[string]any{} err = json.Unmarshal(cseTkgVersionsJson, &versionsMap) if err != nil { - return result, fmt.Errorf("failed unmarshaling cse/tkg_versions.json: %s", err) + return result, fmt.Errorf("failed unmarshaling %s: %s", tkgVersionsMap, err) } versionMap, ok := versionsMap[parsedOvaName] if !ok { - return result, fmt.Errorf("the Kubernetes OVA '%s' is not supported", parsedOvaName) + return result, fmt.Errorf("the Kubernetes Template OVA '%s' is not supported", parsedOvaName) } ovaParts := strings.Split(parsedOvaName, "-") if len(ovaParts) < 2 { - return result, fmt.Errorf("unexpected error parsing the OVA name '%s', it doesn't follow the original naming convention", parsedOvaName) + return result, fmt.Errorf("unexpected error parsing the Kubernetes Template OVA name '%s',"+ + "it doesn't follow the original naming convention (e.g: ubuntu-2004-kube-v1.24.11+vmware.1-tkg.1-2ccb2a001f8bd8f15f1bfbc811071830)", parsedOvaName) } result.KubernetesVersion = ovaParts[0] @@ -705,46 +708,58 @@ func getTkgVersionBundleFromVAppTemplateName(ovaName string) (tkgVersionBundle, return result, nil } -// getVcdKeConfig gets the required information from the CSE Server configuration RDE -func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheckActive bool) (vcdKeConfig, error) { - result := vcdKeConfig{} +// getVcdKeConfig gets the required information from the CSE Server configuration RDE (VCDKEConfig), such as the +// Machine Health Check settings and the Container Registry URL. +func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheckActive bool) (*vcdKeConfig, error) { rdes, err := getRdesByName(client, "vmware", "VCDKEConfig", vcdKeConfigVersion, "vcdKeConfig") if err != nil { - return result, fmt.Errorf("could not retrieve VCDKEConfig RDE with version %s: %s", vcdKeConfigVersion, err) + return nil, err } if len(rdes) != 1 { - return result, fmt.Errorf("expected exactly one VCDKEConfig RDE but got %d", len(rdes)) + return nil, fmt.Errorf("expected exactly one VCDKEConfig RDE with version '%s', but got %d", vcdKeConfigVersion, len(rdes)) } profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]any) if !ok { - return result, fmt.Errorf("wrong format of VCDKEConfig, expected a 'profiles' array") + return nil, fmt.Errorf("wrong format of VCDKEConfig RDE contents, expected a 'profiles' array") } - if len(profiles) != 1 { - return result, fmt.Errorf("wrong format of VCDKEConfig, expected a single 'profiles' element, got %d", len(profiles)) + if len(profiles) == 0 { + return nil, fmt.Errorf("wrong format of VCDKEConfig RDE contents, expected a non-empty 'profiles' element") } + + result := &vcdKeConfig{} // TODO: Check airgapped environments: https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.1.1a/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.1.1/GUID-F00BE796-B5F2-48F2-A012-546E2E694400.html result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]any)["containerRegistryUrl"]) if isNodeHealthCheckActive { - mhc, ok := profiles[0].(map[string]any)["K8Config"].(map[string]any)["mhc"].(map[string]any) + mhc, ok := profiles[0].(map[string]any)["K8Config"].(map[string]any)["mhc"] if !ok { + // If there is no "mhc" entry in the VCDKEConfig JSON, we skip setting this part of the Kubernetes cluster configuration return result, nil } - result.MaxUnhealthyNodesPercentage = mhc["maxUnhealthyNodes"].(float64) - result.NodeStartupTimeout = mhc["nodeStartupTimeout"].(string) - result.NodeNotReadyTimeout = mhc["nodeUnknownTimeout"].(string) - result.NodeUnknownTimeout = mhc["nodeNotReadyTimeout"].(string) + result.MaxUnhealthyNodesPercentage = mhc.(map[string]any)["maxUnhealthyNodes"].(float64) + result.NodeStartupTimeout = mhc.(map[string]any)["nodeStartupTimeout"].(string) + result.NodeNotReadyTimeout = mhc.(map[string]any)["nodeUnknownTimeout"].(string) + result.NodeUnknownTimeout = mhc.(map[string]any)["nodeNotReadyTimeout"].(string) } return result, nil } +// getCseTemplate reads the Go template present in the embedded cseFiles filesystem. func getCseTemplate(cseVersion semver.Version, templateName string) (string, error) { - cseVersionSegments := cseVersion.Segments() - result, err := cseFiles.ReadFile(fmt.Sprintf("cse/%d.%d/%s.tmpl", cseVersionSegments[0], cseVersionSegments[1], templateName)) + minimumVersion, err := semver.NewVersion("4.1") if err != nil { return "", err } + if cseVersion.LessThan(minimumVersion) { + return "", fmt.Errorf("the Container Service version '%s' is not supported", minimumVersion.String()) + } + versionSegments := cseVersion.Segments() + fullTemplatePath := fmt.Sprintf("cse/%d.%d/%s.tmpl", versionSegments[0], versionSegments[1], templateName) + result, err := cseFiles.ReadFile(fullTemplatePath) + if err != nil { + return "", fmt.Errorf("could not read Go template '%s'", fullTemplatePath) + } return string(result), nil } diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index b2e9c66e2..1fd515712 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -195,7 +195,8 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName } // We need to add the block to the slice of YAML documents - mhcYaml, err := generateMemoryHealthCheckYaml(*vcdKeConfig, cseVersion, clusterName) + settings := &cseClusterSettingsInternal{CseVersion: cseVersion, Name: clusterName, VcdKeConfig: *vcdKeConfig} + mhcYaml, err := settings.generateMemoryHealthCheckYaml() if err != nil { return nil, err } @@ -278,7 +279,7 @@ func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateIn if err != nil { return "", err } - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, input.clusterName, input.cseVersion, &vcdKeConfig) + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, input.clusterName, input.cseVersion, vcdKeConfig) if err != nil { return "", err } From 883e67812287ae890b10c0395449b417d961822f Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 6 Feb 2024 13:24:02 +0100 Subject: [PATCH 020/115] Refactor and add unit tests Signed-off-by: abarreiro --- govcd/cse.go | 20 +-- govcd/cse_util.go | 48 +++-- govcd/cse_util_test.go | 7 + govcd/cse_util_unit_test.go | 68 +++++++ govcd/cse_yaml.go | 346 +++++++++++++----------------------- types/v56/cse.go | 271 ++++++++++++++-------------- 6 files changed, 384 insertions(+), 376 deletions(-) create mode 100644 govcd/cse_util_test.go create mode 100644 govcd/cse_util_unit_test.go diff --git a/govcd/cse.go b/govcd/cse.go index 179966e45..810958b2a 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -45,7 +45,7 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) return "", fmt.Errorf("error creating the CSE Kubernetes cluster: %s", err) } - rdeContents, err := internalSettings.getKubernetesClusterCreationPayload() + payload, err := internalSettings.getKubernetesClusterCreationPayload() if err != nil { return "", err } @@ -59,13 +59,13 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) types.DefinedEntity{ EntityType: internalSettings.RdeType.ID, Name: internalSettings.Name, - Entity: rdeContents, + Entity: payload, }, &TenantContext{ OrgId: org.Org.ID, OrgName: org.Org.Name, }) if err != nil { - return "", err + return "", fmt.Errorf("error creating the CSE Kubernetes cluster: %s", err) } return rde.DefinedEntity.ID, nil @@ -123,7 +123,7 @@ func (cluster *CseKubernetesCluster) GetKubeconfig() (string, error) { } // UpdateWorkerPools executes an update on the receiver cluster to change the existing worker pools. -// If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ WorkerPools: &input, @@ -131,7 +131,7 @@ func (cluster *CseKubernetesCluster) UpdateWorkerPools(input map[string]CseWorke } // AddWorkerPools executes an update on the receiver cluster to add new worker pools. -// If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) AddWorkerPools(input []CseWorkerPoolSettings, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ NewWorkerPools: &input, @@ -139,7 +139,7 @@ func (cluster *CseKubernetesCluster) AddWorkerPools(input []CseWorkerPoolSetting } // UpdateControlPlane executes an update on the receiver cluster to change the existing control plane. -// If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpdateInput, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ ControlPlane: &input, @@ -147,7 +147,7 @@ func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpd } // ChangeKubernetesTemplate executes an update on the receiver cluster to change the Kubernetes template of the cluster. -// If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) ChangeKubernetesTemplate(kubernetesTemplateOvaId string, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ KubernetesTemplateOvaId: &kubernetesTemplateOvaId, @@ -155,7 +155,7 @@ func (cluster *CseKubernetesCluster) ChangeKubernetesTemplate(kubernetesTemplate } // SetHealthCheck executes an update on the receiver cluster to enable or disable the machine health check capabilities. -// If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) SetHealthCheck(healthCheckEnabled bool, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ NodeHealthCheck: &healthCheckEnabled, @@ -163,7 +163,7 @@ func (cluster *CseKubernetesCluster) SetHealthCheck(healthCheckEnabled bool, ref } // SetAutoRepairOnErrors executes an update on the receiver cluster to change the flag that controls the auto-repair -// capabilities of CSE. If refresh=true, it retrieves the latest state of the cluster from VCD before updating (recommended). +// capabilities of CSE. If refresh=true, it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) SetAutoRepairOnErrors(autoRepairOnErrors bool, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ AutoRepairOnErrors: &autoRepairOnErrors, @@ -171,7 +171,7 @@ func (cluster *CseKubernetesCluster) SetAutoRepairOnErrors(autoRepairOnErrors bo } // Update executes an update on the receiver CSE Kubernetes Cluster on any of the allowed parameters defined in the input type. If refresh=true, -// it retrieves the latest state of the cluster from VCD before updating (recommended). +// it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh bool) error { if refresh { err := cluster.Refresh() diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 509426725..38e4fc9a5 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -208,7 +208,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu // NOTE: We get the remaining elements from the CAPI YAML, despite they are also inside capvcdType.Status. // The reason is that any change on the cluster is immediately reflected in the CAPI YAML, but not in the capvcdType.Status - // elements, which may take 10 minutes to be refreshed. + // elements, which may take more than 10 minutes to be refreshed. yamlDocuments, err := unmarshalMultipleYamlDocuments(result.capvcdType.Spec.CapiYaml) if err != nil { return nil, err @@ -284,13 +284,24 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu result.ControlPlane.DiskSizeGi = diskSizeGi - // We do it just once for the Control Plane because all VCDMachineTemplate blocks share the same OVA + // We retrieve the Kubernetes Template OVA just once for the Control Plane because all YAML blocks share the same + catalogName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.catalog") + if err != nil { + return nil, err + } + catalog, err := rde.client.GetCatalogByName(result.capvcdType.Status.Capvcd.VcdProperties.Organizations[0].Name, catalogName) + if err != nil { + return nil, err + } ovaName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.template") if err != nil { return nil, err } - // TODO: There is no method to retrieve vApp templates by name.... - result.KubernetesTemplateOvaId = ovaName + ova, err := catalog.GetVAppTemplateByName(ovaName) + if err != nil { + return nil, err + } + result.KubernetesTemplateOvaId = ova.VAppTemplate.ID } else { // This is one Worker Pool. We need to check the map of worker pools, just in case we already saved the // machine count from MachineDeployment. @@ -372,9 +383,8 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu } // waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if timeoutMinutes = 0) -// or until the timeout is reached. If the cluster is in "provisioned" state before the given timeout, it returns a CseKubernetesCluster object -// representing the Kubernetes cluster with all its latest details. -// If one of the states of the cluster at a given point is "error", this function also checks whether the cluster has the "Auto Repair on Errors" flag enabled, +// or until the timeout is reached. +// If one of the states of the cluster at a given point is "error", this function also checks whether the cluster has the "AutoRepairOnErrors" flag enabled, // so it keeps waiting if it's true. // If timeout is reached before the cluster is in "provisioned" state, it returns an error. func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinutes time.Duration) error { @@ -389,15 +399,14 @@ func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinu return err } - // Here we don't to use cseConvertToCseKubernetesClusterType to avoid calling VCD + // Here we don't use cseConvertToCseKubernetesClusterType to avoid calling VCD. We only need the state. entityBytes, err := json.Marshal(rde.DefinedEntity.Entity) if err != nil { - return fmt.Errorf("could not marshal the RDE contents to create a capvcdType instance: %s", err) + return fmt.Errorf("could not check the Kubernetes cluster state: %s", err) } - err = json.Unmarshal(entityBytes, &capvcd) if err != nil { - return fmt.Errorf("could not unmarshal the RDE contents to create a Capvcd instance: %s", err) + return fmt.Errorf("could not check the Kubernetes cluster state: %s", err) } switch capvcd.Status.VcdKe.State { @@ -408,11 +417,10 @@ func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinu if !capvcd.Spec.VcdKe.AutoRepairOnErrors { // Give feedback about what went wrong errors := "" - // TODO: Change to ErrorSet - for _, event := range capvcd.Status.Capvcd.EventSet { - errors += fmt.Sprintf("%s,\n", event.AdditionalDetails) + for _, event := range capvcd.Status.Capvcd.ErrorSet { + errors += fmt.Sprintf("%s,\n", event.AdditionalDetails.DetailedError) } - return fmt.Errorf("got an error and 'AutoRepairOnErrors' is disabled, aborting. Errors:\n%s", errors) + return fmt.Errorf("got an error and 'AutoRepairOnErrors' is disabled, aborting. Error events:\n%s", errors) } } @@ -667,6 +675,9 @@ func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClus // If it is not a valid Kubernetes Template OVA, returns an error. func getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName string) (tkgVersionBundle, error) { result := tkgVersionBundle{} + if strings.TrimSpace(kubernetesTemplateOvaName) == "" { + return result, fmt.Errorf("the Kubernetes Template OVA cannot be empty") + } if strings.Contains(kubernetesTemplateOvaName, "photon") { return result, fmt.Errorf("the Kubernetes Template OVA '%s' uses Photon, and it is not supported", kubernetesTemplateOvaName) @@ -691,13 +702,14 @@ func getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName string) ( } versionMap, ok := versionsMap[parsedOvaName] if !ok { - return result, fmt.Errorf("the Kubernetes Template OVA '%s' is not supported", parsedOvaName) + return result, fmt.Errorf("the Kubernetes Template OVA '%s' is not supported", kubernetesTemplateOvaName) } + // This check should not be needed unless the tkgVersionsMap JSON is deliberately bad constructed. ovaParts := strings.Split(parsedOvaName, "-") - if len(ovaParts) < 2 { + if len(ovaParts) != 3 { return result, fmt.Errorf("unexpected error parsing the Kubernetes Template OVA name '%s',"+ - "it doesn't follow the original naming convention (e.g: ubuntu-2004-kube-v1.24.11+vmware.1-tkg.1-2ccb2a001f8bd8f15f1bfbc811071830)", parsedOvaName) + "it doesn't follow the original naming convention (e.g: ubuntu-2004-kube-v1.24.11+vmware.1-tkg.1-2ccb2a001f8bd8f15f1bfbc811071830)", kubernetesTemplateOvaName) } result.KubernetesVersion = ovaParts[0] diff --git a/govcd/cse_util_test.go b/govcd/cse_util_test.go new file mode 100644 index 000000000..daa5f2889 --- /dev/null +++ b/govcd/cse_util_test.go @@ -0,0 +1,7 @@ +//go:build functional || openapi || cse || ALL + +/* + * Copyright 2024 VMware, Inc. All rights reserved. Licensed under the Apache v2 License. + */ + +package govcd diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go new file mode 100644 index 000000000..17f8c96d6 --- /dev/null +++ b/govcd/cse_util_unit_test.go @@ -0,0 +1,68 @@ +//go:build unit || ALL + +/* + * Copyright 2024 VMware, Inc. All rights reserved. Licensed under the Apache v2 License. + */ + +package govcd + +import ( + "reflect" + "testing" +) + +// Test_getTkgVersionBundleFromVAppTemplateName tests getTkgVersionBundleFromVAppTemplateName function +func Test_getTkgVersionBundleFromVAppTemplateName(t *testing.T) { + tests := []struct { + name string + kubernetesTemplateOvaName string + want tkgVersionBundle + wantErr string + }{ + { + name: "input is empty", + kubernetesTemplateOvaName: "", + wantErr: "the Kubernetes Template OVA cannot be empty", + }, + { + name: "input is Photon OVA", + kubernetesTemplateOvaName: "photon-2004-kube-v9.99.9+vmware.9-tkg.9-aaaaa.ova", + wantErr: "the Kubernetes Template OVA 'photon-2004-kube-v9.99.9+vmware.9-tkg.9-aaaaa.ova' uses Photon, and it is not supported", + }, + { + name: "input is not a Kubernetes OVA", + kubernetesTemplateOvaName: "random-ova.ova", + wantErr: "the OVA 'random-ova.ova' is not a Kubernetes template OVA", + }, + { + name: "input is not supported", + kubernetesTemplateOvaName: "ubuntu-2004-kube-v9.99.9+vmware.9-tkg.9-99999999999999999999999999999999.ova", + wantErr: "the Kubernetes Template OVA 'ubuntu-2004-kube-v9.99.9+vmware.9-tkg.9-99999999999999999999999999999999.ova' is not supported", + }, + { + name: "correct OVA", + kubernetesTemplateOvaName: "ubuntu-2004-kube-v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc.ova", + want: tkgVersionBundle{ + EtcdVersion: "v3.5.6_vmware.9", + CoreDnsVersion: "v1.9.3_vmware.8", + TkgVersion: "v2.2.0", + TkrVersion: "v1.25.7---vmware.2-tkg.1", + KubernetesVersion: "v1.25.7+vmware.2", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getTkgVersionBundleFromVAppTemplateName(tt.kubernetesTemplateOvaName) + if err != nil { + if tt.wantErr != err.Error() { + t.Errorf("getTkgVersionBundleFromVAppTemplateName() error = %v, wantErr = %v", err, tt.wantErr) + } + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getTkgVersionBundleFromVAppTemplateName() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 1fd515712..6d8dd72b5 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -7,53 +7,77 @@ import ( "strings" ) -// traverseMapAndGet traverses the input interface{}, which should be a map of maps, by following the path specified as -// "keyA.keyB.keyC.keyD", doing something similar to, visually speaking, map["keyA"]["keyB"]["keyC"]["keyD"], or in other words, -// it goes inside every inner map, which are inside the initial map, until the given path is finished. -// The final value, "keyD" in the same example, should be of type ResultType, which is a generic type requested during the call -// to this function. -func traverseMapAndGet[ResultType any](input interface{}, path string) (ResultType, error) { - var nothing ResultType - if input == nil { - return nothing, fmt.Errorf("the input is nil") +// cseUpdateCapiYaml takes a YAML and modifies its Kubernetes Template OVA, its Control plane, its Worker pools +// and its Node Health Check capabilities, by using the new values provided as input. +// If some of the values of the input is not provided, it doesn't change them. +// If none of the values is provided, it just returns the same untouched YAML. +func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateInput) (string, error) { + if input.ControlPlane == nil && input.WorkerPools == nil && input.NodeHealthCheck == nil && input.KubernetesTemplateOvaId == nil { + return capiYaml, nil } - inputMap, ok := input.(map[string]any) - if !ok { - return nothing, fmt.Errorf("the input is a %T, not a map[string]interface{}", input) + + // The YAML contains multiple documents, so we cannot use a simple yaml.Unmarshal() as this one just gets the first + // document it finds. + yamlDocs, err := unmarshalMultipleYamlDocuments(capiYaml) + if err != nil { + return capiYaml, fmt.Errorf("error unmarshaling YAML: %s", err) } - if len(inputMap) == 0 { - return nothing, fmt.Errorf("the map is empty") + + // As a side note, we can't optimize this one with "if equals do nothing" because + // in order to retrieve the current value we would need to explore the YAML anyway, which is what we also need to do to update it. + // Also, even if we did it, the current value obtained from YAML would be a Name, but the new value is an ID, so we would need to query VCD anyway + // as well. + // So in this special case this "optimization" would optimize nothing. The same happens with other YAML values. + if input.KubernetesTemplateOvaId != nil { + vAppTemplate, err := getVAppTemplateById(client, *input.KubernetesTemplateOvaId) + if err != nil { + return capiYaml, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", *input.KubernetesTemplateOvaId, err) + } + err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate.Name) + if err != nil { + return capiYaml, err + } } - pathUnits := strings.Split(path, ".") - completed := false - i := 0 - var result interface{} - for !completed { - subPath := pathUnits[i] - traversed, ok := inputMap[subPath] - if !ok { - return nothing, fmt.Errorf("key '%s' does not exist in input map", subPath) + + if input.ControlPlane != nil { + err := cseUpdateControlPlaneInYaml(yamlDocs, *input.ControlPlane) + if err != nil { + return capiYaml, err } - if i < len(pathUnits)-1 { - traversedMap, ok := traversed.(map[string]any) - if !ok { - return nothing, fmt.Errorf("key '%s' is a %T, not a map[string]interface{}, but there are still %d paths to explore", subPath, traversed, len(pathUnits)-(i+1)) - } - inputMap = traversedMap - } else { - completed = true - result = traversed + } + + if input.WorkerPools != nil { + err := cseUpdateWorkerPoolsInYaml(yamlDocs, *input.WorkerPools) + if err != nil { + return capiYaml, err } - i++ } - resultTyped, ok := result.(ResultType) - if !ok { - return nothing, fmt.Errorf("could not convert obtained type %T to requested %T", result, nothing) + + if input.NewWorkerPools != nil { + yamlDocs, err = cseAddWorkerPoolsInYaml(yamlDocs, *input.NewWorkerPools) + if err != nil { + return capiYaml, err + } } - return resultTyped, nil + + if input.NodeHealthCheck != nil { + vcdKeConfig, err := getVcdKeConfig(client, input.vcdKeConfigVersion, *input.NodeHealthCheck) + if err != nil { + return "", err + } + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, input.clusterName, input.cseVersion, vcdKeConfig) + if err != nil { + return "", err + } + } + + return marshalMultipleYamlDocuments(yamlDocs) } -// cseUpdateKubernetesTemplateInYaml updates the Kubernetes template OVA used by all the VCDMachineTemplate blocks +// cseUpdateKubernetesTemplateInYaml modifies the given Kubernetes cluster YAML by modifying the Kubernetes Template OVA +// used by all the cluster elements. +// The caveat here is that not only VCDMachineTemplate needs to be changed with the new OVA name, but also +// other fields that reference the related Kubernetes version, TKG version and other derived information. func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]any, kubernetesTemplateOvaName string) error { tkgBundle, err := getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName) if err != nil { @@ -64,43 +88,40 @@ func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]any, kubernete case "VCDMachineTemplate": _, err := traverseMapAndGet[string](d, "spec.template.spec.template") if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + return fmt.Errorf("incorrect YAML: %s", err) } d["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["template"] = kubernetesTemplateOvaName case "MachineDeployment": _, err := traverseMapAndGet[string](d, "spec.template.spec.version") if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + return fmt.Errorf("incorrect YAML: %s", err) } d["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["version"] = tkgBundle.KubernetesVersion case "Cluster": _, err := traverseMapAndGet[string](d, "metadata.annotations.TKGVERSION") if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + return fmt.Errorf("incorrect YAML: %s", err) } d["metadata"].(map[string]any)["annotations"].(map[string]any)["TKGVERSION"] = tkgBundle.TkgVersion - _, err = traverseMapAndGet[string](d, "metadata.labels.tanzuKubernetesRelease") if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + return fmt.Errorf("incorrect YAML: %s", err) } d["metadata"].(map[string]any)["labels"].(map[string]any)["tanzuKubernetesRelease"] = tkgBundle.TkrVersion case "KubeadmControlPlane": _, err := traverseMapAndGet[string](d, "spec.version") if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + return fmt.Errorf("incorrect YAML: %s", err) } d["spec"].(map[string]any)["version"] = tkgBundle.KubernetesVersion - _, err = traverseMapAndGet[string](d, "spec.kubeadmConfigSpec.clusterConfiguration.dns.imageTag") if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + return fmt.Errorf("incorrect YAML: %s", err) } d["spec"].(map[string]any)["kubeadmConfigSpec"].(map[string]any)["clusterConfiguration"].(map[string]any)["dns"].(map[string]any)["imageTag"] = tkgBundle.CoreDnsVersion - _, err = traverseMapAndGet[string](d, "spec.kubeadmConfigSpec.clusterConfiguration.etcd.local.imageTag") if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + return fmt.Errorf("incorrect YAML: %s", err) } d["spec"].(map[string]any)["kubeadmConfigSpec"].(map[string]any)["clusterConfiguration"].(map[string]any)["etcd"].(map[string]any)["local"].(map[string]any)["imageTag"] = tkgBundle.EtcdVersion } @@ -108,6 +129,7 @@ func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]any, kubernete return nil } +// cseUpdateControlPlaneInYaml modifies the given Kubernetes cluster YAML contents by changing the Control Plane with the input parameters. func cseUpdateControlPlaneInYaml(yamlDocuments []map[string]any, input CseControlPlaneUpdateInput) error { if input.MachineCount < 0 { return fmt.Errorf("incorrect machine count for Control Plane: %d. Should be at least 0", input.MachineCount) @@ -118,20 +140,21 @@ func cseUpdateControlPlaneInYaml(yamlDocuments []map[string]any, input CseContro if d["kind"] != "KubeadmControlPlane" { continue } - _, err := traverseMapAndGet[float64](d, "spec.replicas") if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + return fmt.Errorf("incorrect YAML: %s", err) } d["spec"].(map[string]any)["replicas"] = float64(input.MachineCount) // As it was originally unmarshalled as a float64 updated = true } if !updated { - return fmt.Errorf("could not update the KubeadmControlPlane block in the CAPI YAML") + return fmt.Errorf("could not find the KubeadmControlPlane object in the YAML") } return nil } +// cseUpdateControlPlaneInYaml modifies the given Kubernetes cluster YAML contents by changing +// the existing Worker Pools with the input parameters. func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[string]CseWorkerPoolUpdateInput) error { updated := 0 for _, d := range yamlDocuments { @@ -141,7 +164,7 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[ workerPoolName, err := traverseMapAndGet[string](d, "metadata.name") if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + return fmt.Errorf("incorrect YAML: %s", err) } workerPoolToUpdate := "" @@ -150,7 +173,7 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[ workerPoolToUpdate = wpName } } - // This worker pool is not going to be updated, continue searching for another one + // This worker pool must not be updated as it is not present in the input, continue searching for the ones we want if workerPoolToUpdate == "" { continue } @@ -161,7 +184,7 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[ _, err = traverseMapAndGet[float64](d, "spec.replicas") if err != nil { - return fmt.Errorf("incorrect CAPI YAML: %s", err) + return fmt.Errorf("incorrect YAML: %s", err) } d["spec"].(map[string]any)["replicas"] = float64(workerPools[workerPoolToUpdate].MachineCount) // As it was originally unmarshalled as a float64 @@ -173,10 +196,14 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[ return nil } +// cseAddWorkerPoolsInYaml modifies the given Kubernetes cluster YAML contents by adding new Worker Pools +// described by the input parameters. func cseAddWorkerPoolsInYaml(docs []map[string]any, inputs []CseWorkerPoolSettings) ([]map[string]any, error) { return nil, nil } +// cseUpdateNodeHealthCheckInYaml updates the Kubernetes cluster described in the given YAML documents by adding or removing +// the MachineHealthCheck object. This function doesn't modify the input, but returns a copy of the YAML with the modifications. func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName string, cseVersion semver.Version, vcdKeConfig *vcdKeConfig) ([]map[string]any, error) { mhcPosition := -1 result := make([]map[string]any, len(yamlDocuments)) @@ -222,171 +249,6 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName return result, nil } -// cseUpdateCapiYaml takes a CAPI YAML and modifies its Kubernetes template, its Control plane, its Worker pools -// and its Node Health Check capabilities, by using the new values provided as input. -// If some of the values of the input is not provided, it doesn't change them. -// If none of the values is provided, it just returns the same untouched YAML. -func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateInput) (string, error) { - if input.ControlPlane == nil && input.WorkerPools == nil && input.NodeHealthCheck == nil && input.KubernetesTemplateOvaId == nil { - return capiYaml, nil - } - - // The CAPI YAML contains multiple documents, so we cannot use a simple yaml.Unmarshal() as this one just gets the first - // document it finds. - yamlDocs, err := unmarshalMultipleYamlDocuments(capiYaml) - if err != nil { - return capiYaml, fmt.Errorf("error unmarshaling CAPI YAML: %s", err) - } - - // As a side note, we can't optimize this one with "if equals do nothing" because - // in order to retrieve the current value we would need to explore the YAML anyway, which is what we also need to do to update it. - // So in this special case this "optimization" would optimize nothing. The same happens with other YAML values. - if input.KubernetesTemplateOvaId != nil { - vAppTemplate, err := getVAppTemplateById(client, *input.KubernetesTemplateOvaId) - if err != nil { - return capiYaml, fmt.Errorf("could not retrieve the Kubernetes OVA with ID '%s': %s", *input.KubernetesTemplateOvaId, err) - } - - err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate.Name) - if err != nil { - return capiYaml, err - } - } - - if input.ControlPlane != nil { - err := cseUpdateControlPlaneInYaml(yamlDocs, *input.ControlPlane) - if err != nil { - return capiYaml, err - } - } - - if input.WorkerPools != nil { - err := cseUpdateWorkerPoolsInYaml(yamlDocs, *input.WorkerPools) - if err != nil { - return capiYaml, err - } - } - - if input.NewWorkerPools != nil { - yamlDocs, err = cseAddWorkerPoolsInYaml(yamlDocs, *input.NewWorkerPools) - if err != nil { - return capiYaml, err - } - } - - if input.NodeHealthCheck != nil { - vcdKeConfig, err := getVcdKeConfig(client, input.vcdKeConfigVersion, *input.NodeHealthCheck) - if err != nil { - return "", err - } - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, input.clusterName, input.cseVersion, vcdKeConfig) - if err != nil { - return "", err - } - } - - return marshalMultipleYamlDocuments(yamlDocs) - /* - if d.HasChange("control_plane.0.machine_count") { - for _, yamlDoc := range yamlDocs { - if yamlDoc["kind"] == "KubeadmControlPlane" { - yamlDoc["spec"].(map[string]interface{})["replicas"] = d.Get("control_plane.0.machine_count") - } - } - } - // The node pools can only be created and resized - var newNodePools []map[string]interface{} - if d.HasChange("node_pool") { - for _, nodePoolRaw := range d.Get("node_pool").(*schema.Set).List() { - nodePool := nodePoolRaw.(map[string]interface{}) - for _, yamlDoc := range yamlDocs { - if yamlDoc["kind"] == "MachineDeployment" { - if yamlDoc["metadata"].(map[string]interface{})["name"] == nodePool["name"].(string) { - yamlDoc["spec"].(map[string]interface{})["replicas"] = nodePool["machine_count"].(int) - } else { - // TODO: Create node pool - newNodePools = append(newNodePools, map[string]interface{}{}) - } - } - } - } - } - if len(newNodePools) > 0 { - yamlDocs = append(yamlDocs, newNodePools...) - } - - if d.HasChange("node_health_check") { - oldNhc, newNhc := d.GetChange("node_health_check") - if oldNhc.(bool) && !newNhc.(bool) { - toDelete := 0 - for i, yamlDoc := range yamlDocs { - if yamlDoc["kind"] == "MachineHealthCheck" { - toDelete = i - } - } - yamlDocs[toDelete] = yamlDocs[len(yamlDocs)-1] // We delete the MachineHealthCheck block by putting the last doc in its place - yamlDocs = yamlDocs[:len(yamlDocs)-1] // Then we remove the last doc - } else { - // Add the YAML block - vcdKeConfig, err := getVcdKeConfiguration(d, vcdClient) - if err != nil { - return diag.FromErr(err) - } - rawYaml, err := generateMemoryHealthCheckYaml(d, vcdClient, *vcdKeConfig, d.Get("name").(string)) - if err != nil { - return diag.FromErr(err) - } - yamlBlock := map[string]interface{}{} - err = yaml.Unmarshal([]byte(rawYaml), &yamlBlock) - if err != nil { - return diag.Errorf("error updating Memory Health Check: %s", err) - } - yamlDocs = append(yamlDocs, yamlBlock) - } - util.Logger.Printf("not done but make static complains :)") - } - - updatedYaml, err := yaml.Marshal(yamlDocs) - if err != nil { - return diag.Errorf("error updating cluster: %s", err) - } - - // This must be done with retries due to the possible clash on ETags - _, err = runWithRetry( - "update cluster", - "could not update cluster", - 1*time.Minute, - nil, - func() (any, error) { - rde, err := vcdClient.GetRdeById(d.Id()) - if err != nil { - return nil, fmt.Errorf("could not update Kubernetes cluster with ID '%s': %s", d.Id(), err) - } - - rde.DefinedEntity.Entity["spec"].(map[string]interface{})["capiYaml"] = updatedYaml - rde.DefinedEntity.Entity["spec"].(map[string]interface{})["vcdKe"].(map[string]interface{})["autoRepairOnErrors"] = d.Get("auto_repair_on_errors").(bool) - - // err = rde.Update(*rde.DefinedEntity) - util.Logger.Printf("ADAM: PERFORM UPDATE: %v", rde.DefinedEntity.Entity) - if err != nil { - return nil, err - } - return nil, nil - }, - ) - if err != nil { - return diag.FromErr(err) - } - - state, err = waitUntilClusterIsProvisioned(vcdClient, d, rde.DefinedEntity.ID) - if err != nil { - return diag.Errorf("Kubernetes cluster update failed: %s", err) - } - if state != "provisioned" { - return diag.Errorf("Kubernetes cluster update failed, cluster is not in 'provisioned' state, but '%s'", state) - }*/ -} - // marshalMultipleYamlDocuments takes a slice of maps representing multiple YAML documents (one per item in the slice) and // marshals all of them into a single string with the corresponding separators "---". func marshalMultipleYamlDocuments(yamlDocuments []map[string]any) (string, error) { @@ -422,3 +284,49 @@ func unmarshalMultipleYamlDocuments(yamlDocuments string) ([]map[string]any, err return result, nil } + +// traverseMapAndGet traverses the input interface{}, which should be a map of maps, by following the path specified as +// "keyA.keyB.keyC.keyD", doing something similar to, visually speaking, map["keyA"]["keyB"]["keyC"]["keyD"], or in other words, +// it goes inside every inner map iteratively, until the given path is finished. +// The final value, "keyD" in the same example, should be of type T, which is a generic type requested during the call +// to this function. +func traverseMapAndGet[T any](input interface{}, path string) (T, error) { + var nothing T + if input == nil { + return nothing, fmt.Errorf("the input is nil") + } + inputMap, ok := input.(map[string]any) + if !ok { + return nothing, fmt.Errorf("the input is a %T, not a map[string]interface{}", input) + } + if len(inputMap) == 0 { + return nothing, fmt.Errorf("the map is empty") + } + pathUnits := strings.Split(path, ".") + completed := false + i := 0 + var result interface{} + for !completed { + subPath := pathUnits[i] + traversed, ok := inputMap[subPath] + if !ok { + return nothing, fmt.Errorf("key '%s' does not exist in input map", subPath) + } + if i < len(pathUnits)-1 { + traversedMap, ok := traversed.(map[string]any) + if !ok { + return nothing, fmt.Errorf("key '%s' is a %T, not a map[string]interface{}, but there are still %d paths to explore", subPath, traversed, len(pathUnits)-(i+1)) + } + inputMap = traversedMap + } else { + completed = true + result = traversed + } + i++ + } + resultTyped, ok := result.(T) + if !ok { + return nothing, fmt.Errorf("could not convert obtained type %T to requested %T", result, nothing) + } + return resultTyped, nil +} diff --git a/types/v56/cse.go b/types/v56/cse.go index 1a17a9aa3..42424b005 100644 --- a/types/v56/cse.go +++ b/types/v56/cse.go @@ -5,175 +5,188 @@ import "time" // Capvcd (Cluster API Provider for VCD), is a type that represents a Kubernetes cluster inside VCD, that is created and managed // with the Container Service Extension (CSE) type Capvcd struct { - Kind string `json:"kind"` + Kind string `json:"kind,omitempty"` Spec struct { VcdKe struct { Secure struct { - ApiToken string `json:"apiToken"` - } `json:"secure"` - IsVCDKECluster bool `json:"isVCDKECluster"` - AutoRepairOnErrors bool `json:"autoRepairOnErrors"` + ApiToken string `json:"apiToken,omitempty"` + } `json:"secure,omitempty"` + IsVCDKECluster bool `json:"isVCDKECluster,omitempty"` + AutoRepairOnErrors bool `json:"autoRepairOnErrors,omitempty"` DefaultStorageClassOptions struct { - Filesystem string `json:"filesystem"` - K8SStorageClassName string `json:"k8sStorageClassName"` - VcdStorageProfileName string `json:"vcdStorageProfileName"` - UseDeleteReclaimPolicy bool `json:"useDeleteReclaimPolicy"` - } `json:"defaultStorageClassOptions"` - } `json:"vcdKe"` - CapiYaml string `json:"capiYaml"` - } `json:"spec"` + Filesystem string `json:"filesystem,omitempty"` + K8SStorageClassName string `json:"k8sStorageClassName,omitempty"` + VcdStorageProfileName string `json:"vcdStorageProfileName,omitempty"` + UseDeleteReclaimPolicy bool `json:"useDeleteReclaimPolicy,omitempty"` + } `json:"defaultStorageClassOptions,omitempty"` + } `json:"vcdKe,omitempty"` + CapiYaml string `json:"capiYaml,omitempty"` + } `json:"spec,omitempty"` Status struct { Cpi struct { - Name string `json:"name"` - Version string `json:"version"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` EventSet []struct { - Name string `json:"name"` - OccurredAt time.Time `json:"occurredAt"` - VcdResourceId string `json:"vcdResourceId"` + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` AdditionalDetails struct { - DetailedEvent string `json:"Detailed Event"` - } `json:"additionalDetails"` - } `json:"eventSet"` - } `json:"cpi"` + DetailedEvent string `json:"Detailed Event,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"eventSet,omitempty"` + } `json:"cpi,omitempty"` Csi struct { - Name string `json:"name"` - Version string `json:"version"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` EventSet []struct { - Name string `json:"name"` - OccurredAt time.Time `json:"occurredAt"` + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` AdditionalDetails struct { DetailedDescription string `json:"Detailed Description,omitempty"` - } `json:"additionalDetails"` - } `json:"eventSet"` - } `json:"csi"` + } `json:"additionalDetails,omitempty"` + } `json:"eventSet,omitempty"` + } `json:"csi,omitempty"` VcdKe struct { - State string `json:"state"` + State string `json:"state,omitempty"` EventSet []struct { - Name string `json:"name"` - OccurredAt time.Time `json:"occurredAt"` - VcdResourceId string `json:"vcdResourceId"` + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + AdditionalDetails struct { + DetailedEvent string `json:"Detailed Event,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"eventSet,omitempty"` + ErrorSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` AdditionalDetails struct { - DetailedEvent string `json:"Detailed Event"` - } `json:"additionalDetails"` - } `json:"eventSet"` - WorkerId string `json:"workerId"` - VcdKeVersion string `json:"vcdKeVersion"` + DetailedError string `json:"Detailed Error,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"errorSet,omitempty"` + WorkerId string `json:"workerId,omitempty"` + VcdKeVersion string `json:"vcdKeVersion,omitempty"` VcdResourceSet []struct { - Id string `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - } `json:"vcdResourceSet"` - HeartbeatString string `json:"heartbeatString"` - VcdKeInstanceId string `json:"vcdKeInstanceId"` - HeartbeatTimestamp string `json:"heartbeatTimestamp"` + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + } `json:"vcdResourceSet,omitempty"` + HeartbeatString string `json:"heartbeatString,omitempty"` + VcdKeInstanceId string `json:"vcdKeInstanceId,omitempty"` + HeartbeatTimestamp string `json:"heartbeatTimestamp,omitempty"` DefaultStorageClass struct { - FileSystem string `json:"fileSystem"` - K8SStorageClassName string `json:"k8sStorageClassName"` - VcdStorageProfileName string `json:"vcdStorageProfileName"` - UseDeleteReclaimPolicy bool `json:"useDeleteReclaimPolicy"` - } `json:"defaultStorageClass"` - } `json:"vcdKe"` + FileSystem string `json:"fileSystem,omitempty"` + K8SStorageClassName string `json:"k8sStorageClassName,omitempty"` + VcdStorageProfileName string `json:"vcdStorageProfileName,omitempty"` + UseDeleteReclaimPolicy bool `json:"useDeleteReclaimPolicy,omitempty"` + } `json:"defaultStorageClass,omitempty"` + } `json:"vcdKe,omitempty"` Capvcd struct { - Uid string `json:"uid"` - Phase string `json:"phase"` + Uid string `json:"uid,omitempty"` + Phase string `json:"phase,omitempty"` Private struct { - KubeConfig string `json:"kubeConfig"` - } `json:"private"` + KubeConfig string `json:"kubeConfig,omitempty"` + } `json:"private,omitempty"` Upgrade struct { - Ready bool `json:"ready"` + Ready bool `json:"ready,omitempty"` Current struct { - TkgVersion string `json:"tkgVersion"` - KubernetesVersion string `json:"kubernetesVersion"` - } `json:"current"` - } `json:"upgrade"` + TkgVersion string `json:"tkgVersion,omitempty"` + KubernetesVersion string `json:"kubernetesVersion,omitempty"` + } `json:"current,omitempty"` + } `json:"upgrade,omitempty"` EventSet []struct { - Name string `json:"name"` - OccurredAt time.Time `json:"occurredAt"` - VcdResourceId string `json:"vcdResourceId"` + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` VcdResourceName string `json:"vcdResourceName,omitempty"` AdditionalDetails struct { - Event string `json:"event"` + Event string `json:"event,omitempty"` } `json:"additionalDetails,omitempty"` - } `json:"eventSet"` + } `json:"eventSet,omitempty"` + ErrorSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` + AdditionalDetails struct { + DetailedError string `json:"Detailed Error,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"errorSet,omitempty"` NodePool []struct { - Name string `json:"name"` - DiskSizeMb int `json:"diskSizeMb"` - NodeStatus struct { - CseTest1WorkerNodePool1774Bdcdbffxcwc4BG9Nh9 string `json:"cse-test1-worker-node-pool-1-774bdcdbffxcwc4b-g9nh9,omitempty"` - CseTest1WorkerNodePool1774Bdcdbffxcwc4BRx9Wf string `json:"cse-test1-worker-node-pool-1-774bdcdbffxcwc4b-rx9wf,omitempty"` - CseTest1ControlPlaneNodePool56Jhv string `json:"cse-test1-control-plane-node-pool-56jhv,omitempty"` - } `json:"nodeStatus"` - SizingPolicy string `json:"sizingPolicy"` - StorageProfile string `json:"storageProfile"` - DesiredReplicas int `json:"desiredReplicas"` - AvailableReplicas int `json:"availableReplicas"` - } `json:"nodePool"` - ParentUid string `json:"parentUid"` + Name string `json:"name,omitempty"` + DiskSizeMb int `json:"diskSizeMb,omitempty"` + SizingPolicy string `json:"sizingPolicy,omitempty"` + StorageProfile string `json:"storageProfile,omitempty"` + DesiredReplicas int `json:"desiredReplicas,omitempty"` + AvailableReplicas int `json:"availableReplicas,omitempty"` + } `json:"nodePool,omitempty"` + ParentUid string `json:"parentUid,omitempty"` K8SNetwork struct { Pods struct { - CidrBlocks []string `json:"cidrBlocks"` - } `json:"pods"` + CidrBlocks []string `json:"cidrBlocks,omitempty"` + } `json:"pods,omitempty"` Services struct { - CidrBlocks []string `json:"cidrBlocks"` - } `json:"services"` - } `json:"k8sNetwork"` - Kubernetes string `json:"kubernetes"` - CapvcdVersion string `json:"capvcdVersion"` + CidrBlocks []string `json:"cidrBlocks,omitempty"` + } `json:"services,omitempty"` + } `json:"k8sNetwork,omitempty"` + Kubernetes string `json:"kubernetes,omitempty"` + CapvcdVersion string `json:"capvcdVersion,omitempty"` VcdProperties struct { - Site string `json:"site"` + Site string `json:"site,omitempty"` OrgVdcs []struct { - Id string `json:"id"` - Name string `json:"name"` - OvdcNetworkName string `json:"ovdcNetworkName"` - } `json:"orgVdcs"` + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + OvdcNetworkName string `json:"ovdcNetworkName,omitempty"` + } `json:"orgVdcs,omitempty"` Organizations []struct { - Id string `json:"id"` - Name string `json:"name"` - } `json:"organizations"` - } `json:"vcdProperties"` - CapiStatusYaml string `json:"capiStatusYaml"` + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + } `json:"organizations,omitempty"` + } `json:"vcdProperties,omitempty"` + CapiStatusYaml string `json:"capiStatusYaml,omitempty"` VcdResourceSet []struct { - Id string `json:"id"` - Name string `json:"name"` - Type string `json:"type"` + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` AdditionalDetails struct { - VirtualIP string `json:"virtualIP"` + VirtualIP string `json:"virtualIP,omitempty"` } `json:"additionalDetails,omitempty"` - } `json:"vcdResourceSet"` + } `json:"vcdResourceSet,omitempty"` ClusterApiStatus struct { - Phase string `json:"phase"` + Phase string `json:"phase,omitempty"` ApiEndpoints []struct { - Host string `json:"host"` - Port int `json:"port"` - } `json:"apiEndpoints"` - } `json:"clusterApiStatus"` - CreatedByVersion string `json:"createdByVersion"` + Host string `json:"host,omitempty"` + Port int `json:"port,omitempty"` + } `json:"apiEndpoints,omitempty"` + } `json:"clusterApiStatus,omitempty"` + CreatedByVersion string `json:"createdByVersion,omitempty"` ClusterResourceSetBindings []struct { - Kind string `json:"kind"` - Name string `json:"name"` - Applied bool `json:"applied"` - LastAppliedTime string `json:"lastAppliedTime"` - ClusterResourceSetName string `json:"clusterResourceSetName"` - } `json:"clusterResourceSetBindings"` - } `json:"capvcd"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + Applied bool `json:"applied,omitempty"` + LastAppliedTime string `json:"lastAppliedTime,omitempty"` + ClusterResourceSetName string `json:"clusterResourceSetName,omitempty"` + } `json:"clusterResourceSetBindings,omitempty"` + } `json:"capvcd,omitempty"` Projector struct { - Name string `json:"name"` - Version string `json:"version"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` EventSet []struct { - Name string `json:"name"` - OccurredAt time.Time `json:"occurredAt"` - VcdResourceName string `json:"vcdResourceName"` + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` AdditionalDetails struct { - Event string `json:"event"` - } `json:"additionalDetails"` - } `json:"eventSet"` - } `json:"projector"` - } `json:"status"` + Event string `json:"event,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"eventSet,omitempty"` + } `json:"projector,omitempty"` + } `json:"status,omitempty"` Metadata struct { - Name string `json:"name"` - Site string `json:"site"` - OrgName string `json:"orgName"` - VirtualDataCenterName string `json:"virtualDataCenterName"` - } `json:"metadata"` - ApiVersion string `json:"apiVersion"` + Name string `json:"name,omitempty"` + Site string `json:"site,omitempty"` + OrgName string `json:"orgName,omitempty"` + VirtualDataCenterName string `json:"virtualDataCenterName,omitempty"` + } `json:"metadata,omitempty"` + ApiVersion string `json:"apiVersion,omitempty"` } From 8401bc6ff376a2d4205b7f76ab894c8d60cb35da Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 6 Feb 2024 14:02:51 +0100 Subject: [PATCH 021/115] Add unit test for capiyaml, but it fails Signed-off-by: abarreiro --- govcd/cse_template.go | 14 ++--- govcd/cse_template_unit_test.go | 85 ++++++++++++++++++++++++++++++ govcd/test-resources/capiYaml.yaml | 2 +- 3 files changed, 93 insertions(+), 8 deletions(-) create mode 100644 govcd/cse_template_unit_test.go diff --git a/govcd/cse_template.go b/govcd/cse_template.go index f1cf7030a..29a6e8497 100644 --- a/govcd/cse_template.go +++ b/govcd/cse_template.go @@ -18,7 +18,7 @@ func (clusterSettings *cseClusterSettingsInternal) getKubernetesClusterCreationP if clusterSettings == nil { return nil, fmt.Errorf("the receiver cluster settings is nil") } - capiYaml, err := clusterSettings.generateCapiYaml() + capiYaml, err := clusterSettings.generateCapiYamlAsJsonString() if err != nil { return nil, err } @@ -133,10 +133,10 @@ func (clusterSettings *cseClusterSettingsInternal) generateMemoryHealthCheckYaml if err := mhcEmptyTmpl.Execute(buf, map[string]string{ "ClusterName": clusterSettings.Name, "TargetNamespace": clusterSettings.Name + "-ns", - "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage), // With the 'percentage' suffix - "NodeStartupTimeout": fmt.Sprintf("%ss", clusterSettings.VcdKeConfig.NodeStartupTimeout), // With the 'second' suffix - "NodeUnknownTimeout": fmt.Sprintf("%ss", clusterSettings.VcdKeConfig.NodeUnknownTimeout), // With the 'second' suffix - "NodeNotReadyTimeout": fmt.Sprintf("%ss", clusterSettings.VcdKeConfig.NodeNotReadyTimeout), // With the 'second' suffix + "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage), // With the 'percentage' suffix + "NodeStartupTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeStartupTimeout, "s", "")), // We assure don't duplicate the 's' suffix + "NodeUnknownTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeUnknownTimeout, "s", "")), // We assure don't duplicate the 's' suffix + "NodeNotReadyTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeNotReadyTimeout, "s", "")), // We assure don't duplicate the 's' suffix }); err != nil { return "", fmt.Errorf("could not generate a correct Memory Health Check YAML: %s", err) } @@ -144,10 +144,10 @@ func (clusterSettings *cseClusterSettingsInternal) generateMemoryHealthCheckYaml } -// generateCapiYaml generates the YAML string that is required during Kubernetes cluster creation, to be embedded +// generateCapiYamlAsJsonString generates the YAML string that is required during Kubernetes cluster creation, to be embedded // in the CAPVCD cluster JSON payload. This function picks data from the Terraform schema and the createClusterDto to // populate several Go templates and build a final YAML. -func (clusterSettings *cseClusterSettingsInternal) generateCapiYaml() (string, error) { +func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString() (string, error) { if clusterSettings == nil { return "", fmt.Errorf("the receiver cluster settings is nil") } diff --git a/govcd/cse_template_unit_test.go b/govcd/cse_template_unit_test.go new file mode 100644 index 000000000..fd77dd48e --- /dev/null +++ b/govcd/cse_template_unit_test.go @@ -0,0 +1,85 @@ +//go:build unit || ALL + +package govcd + +import ( + semver "github.com/hashicorp/go-version" + "os" + "reflect" + "strings" + "testing" +) + +// Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString tests cseClusterSettingsInternal.generateCapiYamlAsJsonString +func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { + v41, err := semver.NewVersion("4.1") + if err != nil { + t.Fatalf("%s", err) + } + + capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") + if err != nil { + t.Fatalf("could not read YAML test file: %s", err) + } + expected, err := unmarshalMultipleYamlDocuments(string(capiYaml)) + if err != nil { + t.Fatalf("could not unmarshal YAML test file: %s", err) + } + + clusterSettings := cseClusterSettingsInternal{ + CseVersion: *v41, + Name: "test1", + OrganizationName: "tenant_org", + VdcName: "tenant_vdc", + NetworkName: "tenant_net_routed", + KubernetesTemplateOvaName: "ubuntu-2004-kube-v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc", + TkgVersionBundle: tkgVersionBundle{ + EtcdVersion: "v3.5.6_vmware.9", + CoreDnsVersion: "v1.9.3_vmware.8", + TkgVersion: "v2.2.0", + TkrVersion: "v1.25.7---vmware.2-tkg.1", + KubernetesVersion: "v1.25.7+vmware.2", + }, + CatalogName: "tkgm_catalog", + ControlPlane: cseControlPlaneSettingsInternal{ + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyName: "TKG small", + StorageProfileName: "*", + }, + WorkerPools: []cseWorkerPoolSettingsInternal{ + { + Name: "node-pool-1", + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyName: "TKG small", + StorageProfileName: "*", + }, + }, + VcdKeConfig: vcdKeConfig{ + MaxUnhealthyNodesPercentage: 100, + NodeStartupTimeout: "900", + NodeNotReadyTimeout: "300", + NodeUnknownTimeout: "200", + ContainerRegistryUrl: "projects.registry.vmware.com", + }, + Owner: "dummy", + ApiToken: "dummy", + VcdUrl: "https://www.my-vcd-instance.com", + PodCidr: "100.96.0.0/11", + ServiceCidr: "100.64.0.0/13", + } + got, err := clusterSettings.generateCapiYamlAsJsonString() + if err != nil { + t.Fatalf("generateCapiYamlAsJsonString() failed: %s", err) + } + + gotUnmarshaled, err := unmarshalMultipleYamlDocuments(strings.NewReplacer("\\n", "\n", "\\\"", "\"").Replace(got)) + if err != nil { + t.Fatalf("could not unmarshal obtained YAML: %s", err) + } + + if !reflect.DeepEqual(expected, gotUnmarshaled) { + t.Errorf("generateCapiYamlAsJsonString() got = %v, want %v", gotUnmarshaled, expected) + } +} diff --git a/govcd/test-resources/capiYaml.yaml b/govcd/test-resources/capiYaml.yaml index 1e4a7aacc..180747445 100644 --- a/govcd/test-resources/capiYaml.yaml +++ b/govcd/test-resources/capiYaml.yaml @@ -19,7 +19,7 @@ spec: timeout: "300s" - type: Ready status: "False" - timeout: "300s" + timeout: "200s" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: VCDMachineTemplate From 3e24eb207d2d7808434ee889cd7b2688e569c404 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 6 Feb 2024 17:29:28 +0100 Subject: [PATCH 022/115] Add more tests, change any with interface Signed-off-by: abarreiro --- govcd/cse_template_unit_test.go | 249 +++++++++++++++++++++++------ govcd/cse_util.go | 28 ++-- govcd/cse_yaml.go | 44 ++--- govcd/cse_yaml_unit_test.go | 44 ++--- govcd/test-resources/capiYaml.yaml | 8 +- 5 files changed, 261 insertions(+), 112 deletions(-) diff --git a/govcd/cse_template_unit_test.go b/govcd/cse_template_unit_test.go index fd77dd48e..10cd4a88a 100644 --- a/govcd/cse_template_unit_test.go +++ b/govcd/cse_template_unit_test.go @@ -10,7 +10,6 @@ import ( "testing" ) -// Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString tests cseClusterSettingsInternal.generateCapiYamlAsJsonString func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { v41, err := semver.NewVersion("4.1") if err != nil { @@ -21,65 +20,215 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) if err != nil { t.Fatalf("could not read YAML test file: %s", err) } - expected, err := unmarshalMultipleYamlDocuments(string(capiYaml)) + baseUnmarshaledYaml, err := unmarshalMultipleYamlDocuments(string(capiYaml)) if err != nil { t.Fatalf("could not unmarshal YAML test file: %s", err) } - clusterSettings := cseClusterSettingsInternal{ - CseVersion: *v41, - Name: "test1", - OrganizationName: "tenant_org", - VdcName: "tenant_vdc", - NetworkName: "tenant_net_routed", - KubernetesTemplateOvaName: "ubuntu-2004-kube-v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc", - TkgVersionBundle: tkgVersionBundle{ - EtcdVersion: "v3.5.6_vmware.9", - CoreDnsVersion: "v1.9.3_vmware.8", - TkgVersion: "v2.2.0", - TkrVersion: "v1.25.7---vmware.2-tkg.1", - KubernetesVersion: "v1.25.7+vmware.2", + tests := []struct { + name string + input cseClusterSettingsInternal + expectedFunc func() []map[string]interface{} + wantErr string + }{ + { + name: "correct YAML without optionals", + input: cseClusterSettingsInternal{ + CseVersion: *v41, + Name: "test1", + OrganizationName: "tenant_org", + VdcName: "tenant_vdc", + NetworkName: "tenant_net_routed", + KubernetesTemplateOvaName: "ubuntu-2004-kube-v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc", + TkgVersionBundle: tkgVersionBundle{ + EtcdVersion: "v3.5.6_vmware.9", + CoreDnsVersion: "v1.9.3_vmware.8", + TkgVersion: "v2.2.0", + TkrVersion: "v1.25.7---vmware.2-tkg.1", + KubernetesVersion: "v1.25.7+vmware.2", + }, + CatalogName: "tkgm_catalog", + ControlPlane: cseControlPlaneSettingsInternal{ + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyName: "TKG small", + StorageProfileName: "*", + }, + WorkerPools: []cseWorkerPoolSettingsInternal{ + { + Name: "node-pool-1", + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyName: "TKG small", + StorageProfileName: "*", + }, + }, + VcdKeConfig: vcdKeConfig{ + MaxUnhealthyNodesPercentage: 100, + NodeStartupTimeout: "900", + NodeNotReadyTimeout: "300", + NodeUnknownTimeout: "200", + ContainerRegistryUrl: "projects.registry.vmware.com/tkg", + }, + Owner: "dummy", + ApiToken: "dummy", + VcdUrl: "https://www.my-vcd-instance.com", + PodCidr: "100.96.0.0/11", + ServiceCidr: "100.64.0.0/13", + }, + expectedFunc: func() []map[string]interface{} { + return baseUnmarshaledYaml + }, }, - CatalogName: "tkgm_catalog", - ControlPlane: cseControlPlaneSettingsInternal{ - MachineCount: 1, - DiskSizeGi: 20, - SizingPolicyName: "TKG small", - StorageProfileName: "*", + { + name: "correct YAML without MachineHealthCheck", + input: cseClusterSettingsInternal{ + CseVersion: *v41, + Name: "test1", + OrganizationName: "tenant_org", + VdcName: "tenant_vdc", + NetworkName: "tenant_net_routed", + KubernetesTemplateOvaName: "ubuntu-2004-kube-v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc", + TkgVersionBundle: tkgVersionBundle{ + EtcdVersion: "v3.5.6_vmware.9", + CoreDnsVersion: "v1.9.3_vmware.8", + TkgVersion: "v2.2.0", + TkrVersion: "v1.25.7---vmware.2-tkg.1", + KubernetesVersion: "v1.25.7+vmware.2", + }, + CatalogName: "tkgm_catalog", + ControlPlane: cseControlPlaneSettingsInternal{ + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyName: "TKG small", + StorageProfileName: "*", + }, + WorkerPools: []cseWorkerPoolSettingsInternal{ + { + Name: "node-pool-1", + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyName: "TKG small", + StorageProfileName: "*", + }, + }, + VcdKeConfig: vcdKeConfig{ + ContainerRegistryUrl: "projects.registry.vmware.com/tkg", + }, + Owner: "dummy", + ApiToken: "dummy", + VcdUrl: "https://www.my-vcd-instance.com", + PodCidr: "100.96.0.0/11", + ServiceCidr: "100.64.0.0/13", + }, + expectedFunc: func() []map[string]interface{} { + var result []map[string]interface{} + for _, doc := range baseUnmarshaledYaml { + if doc["kind"] == "MachineHealthCheck" { + continue + } + result = append(result, doc) + } + return result + }, }, - WorkerPools: []cseWorkerPoolSettingsInternal{ - { - Name: "node-pool-1", - MachineCount: 1, - DiskSizeGi: 20, - SizingPolicyName: "TKG small", - StorageProfileName: "*", + { + name: "correct YAML with everything", + input: cseClusterSettingsInternal{ + CseVersion: *v41, + Name: "test1", + OrganizationName: "tenant_org", + VdcName: "tenant_vdc", + NetworkName: "tenant_net_routed", + KubernetesTemplateOvaName: "ubuntu-2004-kube-v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc", + TkgVersionBundle: tkgVersionBundle{ + EtcdVersion: "v3.5.6_vmware.9", + CoreDnsVersion: "v1.9.3_vmware.8", + TkgVersion: "v2.2.0", + TkrVersion: "v1.25.7---vmware.2-tkg.1", + KubernetesVersion: "v1.25.7+vmware.2", + }, + CatalogName: "tkgm_catalog", + ControlPlane: cseControlPlaneSettingsInternal{ + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyName: "TKG small", + StorageProfileName: "*", + Ip: "1.2.3.4", + }, + WorkerPools: []cseWorkerPoolSettingsInternal{ + { + Name: "node-pool-1", + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyName: "TKG small", + StorageProfileName: "*", + }, + }, + VcdKeConfig: vcdKeConfig{ + MaxUnhealthyNodesPercentage: 100, + NodeStartupTimeout: "900", + NodeNotReadyTimeout: "300", + NodeUnknownTimeout: "200", + ContainerRegistryUrl: "projects.registry.vmware.com/tkg", + }, + VirtualIpSubnet: "6.7.8.9/24", + Owner: "dummy", + ApiToken: "dummy", + VcdUrl: "https://www.my-vcd-instance.com", + PodCidr: "100.96.0.0/11", + ServiceCidr: "100.64.0.0/13", + }, + expectedFunc: func() []map[string]interface{} { + var result []map[string]interface{} + for _, doc := range baseUnmarshaledYaml { + if doc["kind"] == "VCDCluster" { + doc["spec"].(map[string]interface{})["controlPlaneEndpoint"] = map[string]interface{}{"host": "1.2.3.4"} + doc["spec"].(map[string]interface{})["controlPlaneEndpoint"].(map[string]interface{})["port"] = 6443 + doc["spec"].(map[string]interface{})["loadBalancerConfigSpec"] = map[string]string{"vipSubnet": "6.7.8.9/24"} + } + result = append(result, doc) + } + return result }, }, - VcdKeConfig: vcdKeConfig{ - MaxUnhealthyNodesPercentage: 100, - NodeStartupTimeout: "900", - NodeNotReadyTimeout: "300", - NodeUnknownTimeout: "200", - ContainerRegistryUrl: "projects.registry.vmware.com", + { + name: "wrong YAML with both Placement and vGPU policies in a Worker Pool", + input: cseClusterSettingsInternal{ + CseVersion: *v41, + WorkerPools: []cseWorkerPoolSettingsInternal{ + { + Name: "node-pool-1", + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyName: "TKG small", + PlacementPolicyName: "policy", + VGpuPolicyName: "policy", + StorageProfileName: "*", + }, + }, + }, + wantErr: "the worker pool 'node-pool-1' should have either a Placement Policy or a vGPU Policy, not both", }, - Owner: "dummy", - ApiToken: "dummy", - VcdUrl: "https://www.my-vcd-instance.com", - PodCidr: "100.96.0.0/11", - ServiceCidr: "100.64.0.0/13", - } - got, err := clusterSettings.generateCapiYamlAsJsonString() - if err != nil { - t.Fatalf("generateCapiYamlAsJsonString() failed: %s", err) - } - - gotUnmarshaled, err := unmarshalMultipleYamlDocuments(strings.NewReplacer("\\n", "\n", "\\\"", "\"").Replace(got)) - if err != nil { - t.Fatalf("could not unmarshal obtained YAML: %s", err) } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.input.generateCapiYamlAsJsonString() + if err != nil { + if err.Error() != tt.wantErr { + t.Errorf("generateCapiYamlAsJsonString() error = %v, wantErr %v", err, tt.wantErr) + } + return + } + gotUnmarshaled, err := unmarshalMultipleYamlDocuments(strings.NewReplacer("\\n", "\n", "\\\"", "\"").Replace(got)) + if err != nil { + t.Fatalf("could not unmarshal obtained YAML: %s", err) + } - if !reflect.DeepEqual(expected, gotUnmarshaled) { - t.Errorf("generateCapiYamlAsJsonString() got = %v, want %v", gotUnmarshaled, expected) + expected := tt.expectedFunc() + if !reflect.DeepEqual(expected, gotUnmarshaled) { + t.Errorf("generateCapiYamlAsJsonString() got =\n%v\nwant =\n%v\n", gotUnmarshaled, expected) + } + }) } } diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 38e4fc9a5..b09798b19 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -227,7 +227,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu } result.ControlPlane.MachineCount = int(replicas) - users, err := traverseMapAndGet[[]any](yamlDocument, "spec.kubeadmConfigSpec.users") + users, err := traverseMapAndGet[[]interface{}](yamlDocument, "spec.kubeadmConfigSpec.users") if err != nil { return nil, err } @@ -350,7 +350,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu result.VirtualIpSubnet = subnet // This is optional } case "Cluster": - cidrBlocks, err := traverseMapAndGet[[]any](yamlDocument, "spec.clusterNetwork.pods.cidrBlocks") + cidrBlocks, err := traverseMapAndGet[[]interface{}](yamlDocument, "spec.clusterNetwork.pods.cidrBlocks") if err != nil { return nil, err } @@ -359,7 +359,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu } result.PodCidr = cidrBlocks[0].(string) - cidrBlocks, err = traverseMapAndGet[[]any](yamlDocument, "spec.clusterNetwork.services.cidrBlocks") + cidrBlocks, err = traverseMapAndGet[[]interface{}](yamlDocument, "spec.clusterNetwork.services.cidrBlocks") if err != nil { return nil, err } @@ -695,7 +695,7 @@ func getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName string) ( return result, fmt.Errorf("failed reading %s: %s", tkgVersionsMap, err) } - versionsMap := map[string]any{} + versionsMap := map[string]interface{}{} err = json.Unmarshal(cseTkgVersionsJson, &versionsMap) if err != nil { return result, fmt.Errorf("failed unmarshaling %s: %s", tkgVersionsMap, err) @@ -714,9 +714,9 @@ func getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName string) ( result.KubernetesVersion = ovaParts[0] result.TkrVersion = strings.ReplaceAll(ovaParts[0], "+", "---") + "-" + ovaParts[1] - result.TkgVersion = versionMap.(map[string]any)["tkg"].(string) - result.EtcdVersion = versionMap.(map[string]any)["etcd"].(string) - result.CoreDnsVersion = versionMap.(map[string]any)["coreDns"].(string) + result.TkgVersion = versionMap.(map[string]interface{})["tkg"].(string) + result.EtcdVersion = versionMap.(map[string]interface{})["etcd"].(string) + result.CoreDnsVersion = versionMap.(map[string]interface{})["coreDns"].(string) return result, nil } @@ -731,7 +731,7 @@ func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheck return nil, fmt.Errorf("expected exactly one VCDKEConfig RDE with version '%s', but got %d", vcdKeConfigVersion, len(rdes)) } - profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]any) + profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]interface{}) if !ok { return nil, fmt.Errorf("wrong format of VCDKEConfig RDE contents, expected a 'profiles' array") } @@ -741,18 +741,18 @@ func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheck result := &vcdKeConfig{} // TODO: Check airgapped environments: https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.1.1a/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.1.1/GUID-F00BE796-B5F2-48F2-A012-546E2E694400.html - result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]any)["containerRegistryUrl"]) + result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"]) if isNodeHealthCheckActive { - mhc, ok := profiles[0].(map[string]any)["K8Config"].(map[string]any)["mhc"] + mhc, ok := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"] if !ok { // If there is no "mhc" entry in the VCDKEConfig JSON, we skip setting this part of the Kubernetes cluster configuration return result, nil } - result.MaxUnhealthyNodesPercentage = mhc.(map[string]any)["maxUnhealthyNodes"].(float64) - result.NodeStartupTimeout = mhc.(map[string]any)["nodeStartupTimeout"].(string) - result.NodeNotReadyTimeout = mhc.(map[string]any)["nodeUnknownTimeout"].(string) - result.NodeUnknownTimeout = mhc.(map[string]any)["nodeNotReadyTimeout"].(string) + result.MaxUnhealthyNodesPercentage = mhc.(map[string]interface{})["maxUnhealthyNodes"].(float64) + result.NodeStartupTimeout = mhc.(map[string]interface{})["nodeStartupTimeout"].(string) + result.NodeNotReadyTimeout = mhc.(map[string]interface{})["nodeUnknownTimeout"].(string) + result.NodeUnknownTimeout = mhc.(map[string]interface{})["nodeNotReadyTimeout"].(string) } return result, nil diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 6d8dd72b5..2d3991a2d 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -78,7 +78,7 @@ func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateIn // used by all the cluster elements. // The caveat here is that not only VCDMachineTemplate needs to be changed with the new OVA name, but also // other fields that reference the related Kubernetes version, TKG version and other derived information. -func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]any, kubernetesTemplateOvaName string) error { +func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]interface{}, kubernetesTemplateOvaName string) error { tkgBundle, err := getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName) if err != nil { return err @@ -90,47 +90,47 @@ func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]any, kubernete if err != nil { return fmt.Errorf("incorrect YAML: %s", err) } - d["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["template"] = kubernetesTemplateOvaName + d["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{})["template"] = kubernetesTemplateOvaName case "MachineDeployment": _, err := traverseMapAndGet[string](d, "spec.template.spec.version") if err != nil { return fmt.Errorf("incorrect YAML: %s", err) } - d["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["version"] = tkgBundle.KubernetesVersion + d["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{})["version"] = tkgBundle.KubernetesVersion case "Cluster": _, err := traverseMapAndGet[string](d, "metadata.annotations.TKGVERSION") if err != nil { return fmt.Errorf("incorrect YAML: %s", err) } - d["metadata"].(map[string]any)["annotations"].(map[string]any)["TKGVERSION"] = tkgBundle.TkgVersion + d["metadata"].(map[string]interface{})["annotations"].(map[string]interface{})["TKGVERSION"] = tkgBundle.TkgVersion _, err = traverseMapAndGet[string](d, "metadata.labels.tanzuKubernetesRelease") if err != nil { return fmt.Errorf("incorrect YAML: %s", err) } - d["metadata"].(map[string]any)["labels"].(map[string]any)["tanzuKubernetesRelease"] = tkgBundle.TkrVersion + d["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["tanzuKubernetesRelease"] = tkgBundle.TkrVersion case "KubeadmControlPlane": _, err := traverseMapAndGet[string](d, "spec.version") if err != nil { return fmt.Errorf("incorrect YAML: %s", err) } - d["spec"].(map[string]any)["version"] = tkgBundle.KubernetesVersion + d["spec"].(map[string]interface{})["version"] = tkgBundle.KubernetesVersion _, err = traverseMapAndGet[string](d, "spec.kubeadmConfigSpec.clusterConfiguration.dns.imageTag") if err != nil { return fmt.Errorf("incorrect YAML: %s", err) } - d["spec"].(map[string]any)["kubeadmConfigSpec"].(map[string]any)["clusterConfiguration"].(map[string]any)["dns"].(map[string]any)["imageTag"] = tkgBundle.CoreDnsVersion + d["spec"].(map[string]interface{})["kubeadmConfigSpec"].(map[string]interface{})["clusterConfiguration"].(map[string]interface{})["dns"].(map[string]interface{})["imageTag"] = tkgBundle.CoreDnsVersion _, err = traverseMapAndGet[string](d, "spec.kubeadmConfigSpec.clusterConfiguration.etcd.local.imageTag") if err != nil { return fmt.Errorf("incorrect YAML: %s", err) } - d["spec"].(map[string]any)["kubeadmConfigSpec"].(map[string]any)["clusterConfiguration"].(map[string]any)["etcd"].(map[string]any)["local"].(map[string]any)["imageTag"] = tkgBundle.EtcdVersion + d["spec"].(map[string]interface{})["kubeadmConfigSpec"].(map[string]interface{})["clusterConfiguration"].(map[string]interface{})["etcd"].(map[string]interface{})["local"].(map[string]interface{})["imageTag"] = tkgBundle.EtcdVersion } } return nil } // cseUpdateControlPlaneInYaml modifies the given Kubernetes cluster YAML contents by changing the Control Plane with the input parameters. -func cseUpdateControlPlaneInYaml(yamlDocuments []map[string]any, input CseControlPlaneUpdateInput) error { +func cseUpdateControlPlaneInYaml(yamlDocuments []map[string]interface{}, input CseControlPlaneUpdateInput) error { if input.MachineCount < 0 { return fmt.Errorf("incorrect machine count for Control Plane: %d. Should be at least 0", input.MachineCount) } @@ -144,7 +144,7 @@ func cseUpdateControlPlaneInYaml(yamlDocuments []map[string]any, input CseContro if err != nil { return fmt.Errorf("incorrect YAML: %s", err) } - d["spec"].(map[string]any)["replicas"] = float64(input.MachineCount) // As it was originally unmarshalled as a float64 + d["spec"].(map[string]interface{})["replicas"] = float64(input.MachineCount) // As it was originally unmarshalled as a float64 updated = true } if !updated { @@ -155,7 +155,7 @@ func cseUpdateControlPlaneInYaml(yamlDocuments []map[string]any, input CseContro // cseUpdateControlPlaneInYaml modifies the given Kubernetes cluster YAML contents by changing // the existing Worker Pools with the input parameters. -func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[string]CseWorkerPoolUpdateInput) error { +func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]interface{}, workerPools map[string]CseWorkerPoolUpdateInput) error { updated := 0 for _, d := range yamlDocuments { if d["kind"] != "MachineDeployment" { @@ -187,7 +187,7 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[ return fmt.Errorf("incorrect YAML: %s", err) } - d["spec"].(map[string]any)["replicas"] = float64(workerPools[workerPoolToUpdate].MachineCount) // As it was originally unmarshalled as a float64 + d["spec"].(map[string]interface{})["replicas"] = float64(workerPools[workerPoolToUpdate].MachineCount) // As it was originally unmarshalled as a float64 updated++ } if updated != len(workerPools) { @@ -198,15 +198,15 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]any, workerPools map[ // cseAddWorkerPoolsInYaml modifies the given Kubernetes cluster YAML contents by adding new Worker Pools // described by the input parameters. -func cseAddWorkerPoolsInYaml(docs []map[string]any, inputs []CseWorkerPoolSettings) ([]map[string]any, error) { +func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, inputs []CseWorkerPoolSettings) ([]map[string]interface{}, error) { return nil, nil } // cseUpdateNodeHealthCheckInYaml updates the Kubernetes cluster described in the given YAML documents by adding or removing // the MachineHealthCheck object. This function doesn't modify the input, but returns a copy of the YAML with the modifications. -func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName string, cseVersion semver.Version, vcdKeConfig *vcdKeConfig) ([]map[string]any, error) { +func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, clusterName string, cseVersion semver.Version, vcdKeConfig *vcdKeConfig) ([]map[string]interface{}, error) { mhcPosition := -1 - result := make([]map[string]any, len(yamlDocuments)) + result := make([]map[string]interface{}, len(yamlDocuments)) for i, d := range yamlDocuments { if d["kind"] == "MachineHealthCheck" { mhcPosition = i @@ -227,7 +227,7 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName if err != nil { return nil, err } - var mhc map[string]any + var mhc map[string]interface{} err = yaml.Unmarshal([]byte(mhcYaml), &mhc) if err != nil { return nil, err @@ -251,7 +251,7 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]any, clusterName // marshalMultipleYamlDocuments takes a slice of maps representing multiple YAML documents (one per item in the slice) and // marshals all of them into a single string with the corresponding separators "---". -func marshalMultipleYamlDocuments(yamlDocuments []map[string]any) (string, error) { +func marshalMultipleYamlDocuments(yamlDocuments []map[string]interface{}) (string, error) { result := "" for i, yamlDoc := range yamlDocuments { updatedSingleDoc, err := yaml.Marshal(yamlDoc) @@ -268,13 +268,13 @@ func marshalMultipleYamlDocuments(yamlDocuments []map[string]any) (string, error // unmarshalMultipleYamlDocuments takes a multi-document YAML (multiple YAML documents are separated by "---") and // unmarshals all of them into a slice of generic maps with the corresponding content. -func unmarshalMultipleYamlDocuments(yamlDocuments string) ([]map[string]any, error) { +func unmarshalMultipleYamlDocuments(yamlDocuments string) ([]map[string]interface{}, error) { if len(strings.TrimSpace(yamlDocuments)) == 0 { - return []map[string]any{}, nil + return []map[string]interface{}{}, nil } splitYamlDocs := strings.Split(yamlDocuments, "---\n") - result := make([]map[string]any, len(splitYamlDocs)) + result := make([]map[string]interface{}, len(splitYamlDocs)) for i, yamlDoc := range splitYamlDocs { err := yaml.Unmarshal([]byte(yamlDoc), &result[i]) if err != nil { @@ -295,7 +295,7 @@ func traverseMapAndGet[T any](input interface{}, path string) (T, error) { if input == nil { return nothing, fmt.Errorf("the input is nil") } - inputMap, ok := input.(map[string]any) + inputMap, ok := input.(map[string]interface{}) if !ok { return nothing, fmt.Errorf("the input is a %T, not a map[string]interface{}", input) } @@ -313,7 +313,7 @@ func traverseMapAndGet[T any](input interface{}, path string) (T, error) { return nothing, fmt.Errorf("key '%s' does not exist in input map", subPath) } if i < len(pathUnits)-1 { - traversedMap, ok := traversed.(map[string]any) + traversedMap, ok := traversed.(map[string]interface{}) if !ok { return nothing, fmt.Errorf("key '%s' is a %T, not a map[string]interface{}, but there are still %d paths to explore", subPath, traversed, len(pathUnits)-(i+1)) } diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index d3bc96436..797fe9dbd 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -382,8 +382,8 @@ func Test_marshalMultplieYamlDocuments(t *testing.T) { tests := []struct { name string - yamlDocuments []map[string]any - want []map[string]any + yamlDocuments []map[string]interface{} + want []map[string]interface{} wantErr bool }{ { @@ -394,8 +394,8 @@ func Test_marshalMultplieYamlDocuments(t *testing.T) { }, { name: "marshal empty slice", - yamlDocuments: []map[string]any{}, - want: []map[string]any{}, + yamlDocuments: []map[string]interface{}{}, + want: []map[string]interface{}{}, wantErr: false, }, } @@ -443,19 +443,19 @@ func Test_traverseMapAndGet(t *testing.T) { args: args{ input: "error", }, - wantErr: "the input is a string, not a map[string]any", + wantErr: "the input is a string, not a map[string]interface{}", }, { name: "map is empty", args: args{ - input: map[string]any{}, + input: map[string]interface{}{}, }, wantErr: "the map is empty", }, { name: "map does not have key", args: args{ - input: map[string]any{ + input: map[string]interface{}{ "keyA": "value", }, path: "keyB", @@ -465,7 +465,7 @@ func Test_traverseMapAndGet(t *testing.T) { { name: "map has a single simple key", args: args{ - input: map[string]any{ + input: map[string]interface{}{ "keyA": "value", }, path: "keyA", @@ -476,24 +476,24 @@ func Test_traverseMapAndGet(t *testing.T) { { name: "map has a single complex key", args: args{ - input: map[string]any{ - "keyA": map[string]any{ + input: map[string]interface{}{ + "keyA": map[string]interface{}{ "keyB": "value", }, }, path: "keyA", }, wantType: "map", - want: map[string]any{ + want: map[string]interface{}{ "keyB": "value", }, }, { name: "map has a complex structure", args: args{ - input: map[string]any{ - "keyA": map[string]any{ - "keyB": map[string]any{ + input: map[string]interface{}{ + "keyA": map[string]interface{}{ + "keyB": map[string]interface{}{ "keyC": "value", }, }, @@ -506,9 +506,9 @@ func Test_traverseMapAndGet(t *testing.T) { { name: "requested path is deeper than the map structure", args: args{ - input: map[string]any{ - "keyA": map[string]any{ - "keyB": map[string]any{ + input: map[string]interface{}{ + "keyA": map[string]interface{}{ + "keyB": map[string]interface{}{ "keyC": "value", }, }, @@ -520,10 +520,10 @@ func Test_traverseMapAndGet(t *testing.T) { { name: "obtained value does not correspond to the desired type", args: args{ - input: map[string]any{ - "keyA": map[string]any{ - "keyB": map[string]any{ - "keyC": map[string]any{}, + input: map[string]interface{}{ + "keyA": map[string]interface{}{ + "keyB": map[string]interface{}{ + "keyC": map[string]interface{}{}, }, }, }, @@ -540,7 +540,7 @@ func Test_traverseMapAndGet(t *testing.T) { if tt.wantType == "string" { got, err = traverseMapAndGet[string](tt.args.input, tt.args.path) } else if tt.wantType == "map" { - got, err = traverseMapAndGet[map[string]any](tt.args.input, tt.args.path) + got, err = traverseMapAndGet[map[string]interface{}](tt.args.input, tt.args.path) } else { t.Fatalf("wantType type not used in this test") } diff --git a/govcd/test-resources/capiYaml.yaml b/govcd/test-resources/capiYaml.yaml index 180747445..762a8a690 100644 --- a/govcd/test-resources/capiYaml.yaml +++ b/govcd/test-resources/capiYaml.yaml @@ -16,10 +16,10 @@ spec: unhealthyConditions: - type: Ready status: Unknown - timeout: "300s" + timeout: "200s" - type: Ready status: "False" - timeout: "200s" + timeout: "300s" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: VCDMachineTemplate @@ -103,8 +103,8 @@ metadata: namespace: test1-ns type: Opaque data: - username: "ZHVtbXkK" - refreshToken: "ZHVtbXkK" + username: "ZHVtbXk=" + refreshToken: "ZHVtbXk=" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: VCDCluster From bf80795021b35b598c65f2df397e6ad122d911f3 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 6 Feb 2024 17:52:03 +0100 Subject: [PATCH 023/115] Add more tests, change any with interface Signed-off-by: abarreiro --- govcd/cse_template_unit_test.go | 2 +- govcd/cse_yaml.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/govcd/cse_template_unit_test.go b/govcd/cse_template_unit_test.go index 10cd4a88a..286e16907 100644 --- a/govcd/cse_template_unit_test.go +++ b/govcd/cse_template_unit_test.go @@ -133,7 +133,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) }, }, { - name: "correct YAML with everything", + name: "correct YAML with every possible option", input: cseClusterSettingsInternal{ CseVersion: *v41, Name: "test1", diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 2d3991a2d..8499d2041 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -288,8 +288,7 @@ func unmarshalMultipleYamlDocuments(yamlDocuments string) ([]map[string]interfac // traverseMapAndGet traverses the input interface{}, which should be a map of maps, by following the path specified as // "keyA.keyB.keyC.keyD", doing something similar to, visually speaking, map["keyA"]["keyB"]["keyC"]["keyD"], or in other words, // it goes inside every inner map iteratively, until the given path is finished. -// The final value, "keyD" in the same example, should be of type T, which is a generic type requested during the call -// to this function. +// The final value, "keyD" in the same example, should be of any type T. func traverseMapAndGet[T any](input interface{}, path string) (T, error) { var nothing T if input == nil { From 3cf8fc344cc1e0b2a3a39fc40b99b5e00842f749 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 6 Feb 2024 18:13:17 +0100 Subject: [PATCH 024/115] Implement cseAddWorkerPoolsInYaml, not teste Signed-off-by: abarreiro --- govcd/cse.go | 2 +- govcd/cse_yaml.go | 62 +++++++++++++++++++++++++++++++++++------------ 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 810958b2a..5f30e0abd 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -195,7 +195,7 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh input.clusterName = cluster.Name input.vcdKeConfigVersion = cluster.capvcdType.Status.VcdKe.VcdKeVersion input.cseVersion = cluster.CseVersion - updatedCapiYaml, err := cseUpdateCapiYaml(cluster.client, cluster.capvcdType.Spec.CapiYaml, input) + updatedCapiYaml, err := cluster.updateCapiYaml(input) if err != nil { return err } diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 8499d2041..9ba263d60 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -7,20 +7,23 @@ import ( "strings" ) -// cseUpdateCapiYaml takes a YAML and modifies its Kubernetes Template OVA, its Control plane, its Worker pools +// updateCapiYaml takes a YAML and modifies its Kubernetes Template OVA, its Control plane, its Worker pools // and its Node Health Check capabilities, by using the new values provided as input. // If some of the values of the input is not provided, it doesn't change them. // If none of the values is provided, it just returns the same untouched YAML. -func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateInput) (string, error) { +func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) (string, error) { + if cluster == nil { + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("receiver cluster is nil") + } if input.ControlPlane == nil && input.WorkerPools == nil && input.NodeHealthCheck == nil && input.KubernetesTemplateOvaId == nil { - return capiYaml, nil + return cluster.capvcdType.Spec.CapiYaml, nil } // The YAML contains multiple documents, so we cannot use a simple yaml.Unmarshal() as this one just gets the first // document it finds. - yamlDocs, err := unmarshalMultipleYamlDocuments(capiYaml) + yamlDocs, err := unmarshalMultipleYamlDocuments(cluster.capvcdType.Spec.CapiYaml) if err != nil { - return capiYaml, fmt.Errorf("error unmarshaling YAML: %s", err) + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("error unmarshaling YAML: %s", err) } // As a side note, we can't optimize this one with "if equals do nothing" because @@ -29,39 +32,39 @@ func cseUpdateCapiYaml(client *Client, capiYaml string, input CseClusterUpdateIn // as well. // So in this special case this "optimization" would optimize nothing. The same happens with other YAML values. if input.KubernetesTemplateOvaId != nil { - vAppTemplate, err := getVAppTemplateById(client, *input.KubernetesTemplateOvaId) + vAppTemplate, err := getVAppTemplateById(cluster.client, *input.KubernetesTemplateOvaId) if err != nil { - return capiYaml, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", *input.KubernetesTemplateOvaId, err) + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", *input.KubernetesTemplateOvaId, err) } err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate.Name) if err != nil { - return capiYaml, err + return cluster.capvcdType.Spec.CapiYaml, err } } if input.ControlPlane != nil { err := cseUpdateControlPlaneInYaml(yamlDocs, *input.ControlPlane) if err != nil { - return capiYaml, err + return cluster.capvcdType.Spec.CapiYaml, err } } if input.WorkerPools != nil { err := cseUpdateWorkerPoolsInYaml(yamlDocs, *input.WorkerPools) if err != nil { - return capiYaml, err + return cluster.capvcdType.Spec.CapiYaml, err } } if input.NewWorkerPools != nil { - yamlDocs, err = cseAddWorkerPoolsInYaml(yamlDocs, *input.NewWorkerPools) + yamlDocs, err = cseAddWorkerPoolsInYaml(yamlDocs, *cluster, *input.NewWorkerPools) if err != nil { - return capiYaml, err + return cluster.capvcdType.Spec.CapiYaml, err } } if input.NodeHealthCheck != nil { - vcdKeConfig, err := getVcdKeConfig(client, input.vcdKeConfigVersion, *input.NodeHealthCheck) + vcdKeConfig, err := getVcdKeConfig(cluster.client, input.vcdKeConfigVersion, *input.NodeHealthCheck) if err != nil { return "", err } @@ -198,8 +201,37 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]interface{}, workerPo // cseAddWorkerPoolsInYaml modifies the given Kubernetes cluster YAML contents by adding new Worker Pools // described by the input parameters. -func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, inputs []CseWorkerPoolSettings) ([]map[string]interface{}, error) { - return nil, nil +func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernetesCluster, newWorkerPools []CseWorkerPoolSettings) ([]map[string]interface{}, error) { + internalSettings := cseClusterSettingsInternal{WorkerPools: make([]cseWorkerPoolSettingsInternal, len(newWorkerPools))} + for i, workerPool := range newWorkerPools { + internalSettings.WorkerPools[i] = cseWorkerPoolSettingsInternal{ + Name: workerPool.Name, + MachineCount: workerPool.MachineCount, + DiskSizeGi: workerPool.DiskSizeGi, + SizingPolicyName: workerPool.SizingPolicyId, + PlacementPolicyName: workerPool.PlacementPolicyId, + VGpuPolicyName: workerPool.VGpuPolicyId, + StorageProfileName: workerPool.StorageProfileId, + } + } + internalSettings.Name = cluster.Name + internalSettings.CseVersion = cluster.CseVersion + nodePoolsYaml, err := internalSettings.generateNodePoolYaml() + if err != nil { + return nil, err + } + + newWorkerPoolsYamlDocs, err := unmarshalMultipleYamlDocuments(nodePoolsYaml) + if err != nil { + return nil, err + } + + result := make([]map[string]interface{}, len(docs)+len(newWorkerPoolsYamlDocs)) + copy(result, docs) + for i, doc := range newWorkerPoolsYamlDocs { + result[i+len(docs)] = doc + } + return result, nil } // cseUpdateNodeHealthCheckInYaml updates the Kubernetes cluster described in the given YAML documents by adding or removing From e113aaa1560252b745cbdb81dc00b981e71cbd7f Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 7 Feb 2024 13:00:51 +0100 Subject: [PATCH 025/115] Fix unit test Signed-off-by: abarreiro --- govcd/cse_yaml.go | 10 ++--- govcd/cse_yaml_unit_test.go | 80 ++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 9ba263d60..8f4eee0e9 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -201,6 +201,7 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]interface{}, workerPo // cseAddWorkerPoolsInYaml modifies the given Kubernetes cluster YAML contents by adding new Worker Pools // described by the input parameters. +// NOTE: This function doesn't modify the input, but returns a copy of the YAML with the added unmarshalled documents. func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernetesCluster, newWorkerPools []CseWorkerPoolSettings) ([]map[string]interface{}, error) { internalSettings := cseClusterSettingsInternal{WorkerPools: make([]cseWorkerPoolSettingsInternal, len(newWorkerPools))} for i, workerPool := range newWorkerPools { @@ -226,16 +227,15 @@ func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernete return nil, err } - result := make([]map[string]interface{}, len(docs)+len(newWorkerPoolsYamlDocs)) + result := make([]map[string]interface{}, len(docs)) copy(result, docs) - for i, doc := range newWorkerPoolsYamlDocs { - result[i+len(docs)] = doc - } + result = append(result, newWorkerPoolsYamlDocs...) return result, nil } // cseUpdateNodeHealthCheckInYaml updates the Kubernetes cluster described in the given YAML documents by adding or removing -// the MachineHealthCheck object. This function doesn't modify the input, but returns a copy of the YAML with the modifications. +// the MachineHealthCheck object. +// NOTE: This function doesn't modify the input, but returns a copy of the YAML with the modifications. func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, clusterName string, cseVersion semver.Version, vcdKeConfig *vcdKeConfig) ([]map[string]interface{}, error) { mhcPosition := -1 result := make([]map[string]interface{}, len(yamlDocuments)) diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 797fe9dbd..937892db3 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -114,7 +114,7 @@ func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { MachineCount: int(oldReplicas), } } - if len(oldNodePools) == -1 { + if len(oldNodePools) == 0 { t.Fatalf("didn't get any valid worker node pool") } @@ -169,6 +169,84 @@ func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { } } +// Test_cseAddWorkerPoolsInYaml tests the addition process of the Worker pools in a CAPI YAML. +func Test_cseAddWorkerPoolsInYaml(t *testing.T) { + version, err := semver.NewVersion("4.1") + if err != nil { + t.Fatalf("could not create version: %s", err) + } + capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") + if err != nil { + t.Fatalf("could not read CAPI YAML test file: %s", err) + } + + yamlDocs, err := unmarshalMultipleYamlDocuments(string(capiYaml)) + if err != nil { + t.Fatalf("could not unmarshal CAPI YAML test file: %s", err) + } + + // The worker pools should have now the new details updated + poolCount := 0 + for _, document := range yamlDocs { + if document["kind"] != "MachineDeployment" { + continue + } + poolCount++ + } + + // We call the function to update the old pools with the new ones + newNodePools := []CseWorkerPoolSettings{{ + Name: "new-pool", + MachineCount: 35, + DiskSizeGi: 20, + SizingPolicyId: "dummy", + PlacementPolicyId: "", + VGpuPolicyId: "", + StorageProfileId: "*", + }} + + newYamlDocs, err := cseAddWorkerPoolsInYaml(yamlDocs, CseKubernetesCluster{ + CseClusterSettings: CseClusterSettings{ + CseVersion: *version, + Name: "dummy", + }, + }, newNodePools) + if err != nil { + t.Fatalf("%s", err) + } + + // The worker pools should have now the new details updated + var newPool map[string]interface{} + newPoolCount := 0 + for _, document := range newYamlDocs { + if document["kind"] != "MachineDeployment" { + continue + } + + name, err := traverseMapAndGet[string](document, "metadata.name") + if err != nil { + t.Fatalf("incorrect CAPI YAML: %s", err) + } + if name == "new-pool" { + newPool = document + } + newPoolCount++ + } + if newPool == nil { + t.Fatalf("should have found the new Worker Pool") + } + if poolCount != newPoolCount-1 { + t.Fatalf("should have one extra Worker Pool") + } + replicas, err := traverseMapAndGet[float64](newPool, "spec.replicas") + if err != nil { + t.Fatalf("incorrect CAPI YAML: %s", err) + } + if replicas != 35 { + t.Fatalf("incorrect replicas: %.f", replicas) + } +} + // Test_cseUpdateControlPlaneInYaml tests the update process of the Control Plane in a CAPI YAML. func Test_cseUpdateControlPlaneInYaml(t *testing.T) { capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") From d997cedf0370d353ee40c17d937ac844ffa774ff Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 7 Feb 2024 13:01:09 +0100 Subject: [PATCH 026/115] Fix unit test Signed-off-by: abarreiro --- govcd/cse_yaml.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 8f4eee0e9..8fc87710b 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -229,8 +229,7 @@ func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernete result := make([]map[string]interface{}, len(docs)) copy(result, docs) - result = append(result, newWorkerPoolsYamlDocs...) - return result, nil + return append(result, newWorkerPoolsYamlDocs...), nil } // cseUpdateNodeHealthCheckInYaml updates the Kubernetes cluster described in the given YAML documents by adding or removing From 7907bb9b99a5d616a08851f725ff348c820600df Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 8 Feb 2024 15:57:26 +0100 Subject: [PATCH 027/115] Refactor traverseMap Signed-off-by: abarreiro --- govcd/cse.go | 8 +++- govcd/cse_test.go | 21 ++++++++- govcd/cse_util.go | 91 ++++++++--------------------------- govcd/cse_util_test.go | 7 --- govcd/cse_util_unit_test.go | 41 ++++++++++++++++ govcd/cse_yaml.go | 75 +++++++++++++---------------- govcd/cse_yaml_unit_test.go | 94 +++++++++++++------------------------ 7 files changed, 151 insertions(+), 186 deletions(-) delete mode 100644 govcd/cse_util_test.go diff --git a/govcd/cse.go b/govcd/cse.go index 5f30e0abd..6fb313e7c 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -97,6 +97,10 @@ func (cluster *CseKubernetesCluster) Refresh() error { // GetKubeconfig retrieves the Kubeconfig from an available cluster. func (cluster *CseKubernetesCluster) GetKubeconfig() (string, error) { + if cluster.State != "provisioned" { + return "", fmt.Errorf("cannot get a Kubeconfig of a Kubernetes cluster that is not in 'provisioned' state") + } + rde, err := getRdeById(cluster.client, cluster.ID) if err != nil { return "", err @@ -146,9 +150,9 @@ func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpd }, refresh) } -// ChangeKubernetesTemplate executes an update on the receiver cluster to change the Kubernetes template of the cluster. +// ChangeKubernetesTemplateOva executes an update on the receiver cluster to change the Kubernetes template of the cluster. // If refresh=true, it retrieves the latest state of the cluster from VCD before updating. -func (cluster *CseKubernetesCluster) ChangeKubernetesTemplate(kubernetesTemplateOvaId string, refresh bool) error { +func (cluster *CseKubernetesCluster) ChangeKubernetesTemplateOva(kubernetesTemplateOvaId string, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ KubernetesTemplateOvaId: &kubernetesTemplateOvaId, }, refresh) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index eafce1bd5..6005ab9f7 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -185,7 +185,7 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { check.Assert(true, Equals, strings.Contains(kubeconfig, "client-key-data")) // Perform the update - err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 1}}, true) + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, true) check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up @@ -193,9 +193,26 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { for _, nodePool := range cluster.WorkerPools { if nodePool.Name == workerPoolName { foundWorkerPool = true - check.Assert(nodePool.MachineCount, Equals, 1) + check.Assert(nodePool.MachineCount, Equals, 2) } } check.Assert(foundWorkerPool, Equals, true) + // Revert back (resources can be limited) + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 1}}, true) + check.Assert(err, IsNil) + + // Perform the update + err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 2}, true) + check.Assert(err, IsNil) + + // Post-check. This should be 2, as it should have scaled up + check.Assert(cluster.ControlPlane.MachineCount, Equals, 2) + + // Revert back (resources can be limited) + err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) + check.Assert(err, IsNil) + + err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) + check.Assert(err, IsNil) } diff --git a/govcd/cse_util.go b/govcd/cse_util.go index b09798b19..4216f0b38 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -14,8 +14,7 @@ import ( "time" ) -// getCseComponentsVersions gets the versions of the sub-components that are part of Container Service Extension. -// TODO: Is this really necessary? What happens in UI if I have a 1.1.0-1.2.0-1.0.0 (4.2) cluster and then CSE is updated to 4.3? +// getCseComponentsVersions gets the versions of the subcomponents that are part of Container Service Extension. func getCseComponentsVersions(cseVersion semver.Version) (*cseComponentsVersions, error) { v41, _ := semver.NewVersion("4.1") @@ -119,7 +118,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu } result.VdcId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Id - // FIXME: This is a workaround, because for some reason the ID contains the VDC name instead of the VDC ID. + // FIXME: This is a workaround, because for some reason the OrgVdcs[*].Id property contains the VDC name instead of the VDC ID. // Once this is fixed, this conditional should not be needed anymore. if result.VdcId == result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Name { vdcs, err := queryOrgVdcList(rde.client, map[string]string{}) @@ -221,48 +220,21 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu for _, yamlDocument := range yamlDocuments { switch yamlDocument["kind"] { case "KubeadmControlPlane": - replicas, err := traverseMapAndGet[float64](yamlDocument, "spec.replicas") - if err != nil { - return nil, err - } - result.ControlPlane.MachineCount = int(replicas) - - users, err := traverseMapAndGet[[]interface{}](yamlDocument, "spec.kubeadmConfigSpec.users") - if err != nil { - return nil, err - } + result.ControlPlane.MachineCount = int(traverseMapAndGet[float64](yamlDocument, "spec.replicas")) + users := traverseMapAndGet[[]interface{}](yamlDocument, "spec.kubeadmConfigSpec.users") if len(users) == 0 { return nil, fmt.Errorf("expected 'spec.kubeadmConfigSpec.users' slice to not to be empty") } - keys, err := traverseMapAndGet[[]string](users[0], "sshAuthorizedKeys") - if err != nil && !strings.Contains(err.Error(), "key 'sshAuthorizedKeys' does not exist in input map") { - return nil, err - } + keys := traverseMapAndGet[[]string](users[0], "sshAuthorizedKeys") if len(keys) > 0 { result.SshPublicKey = keys[0] // Optional field } case "VCDMachineTemplate": - name, err := traverseMapAndGet[string](yamlDocument, "metadata.name") - if err != nil { - return nil, err - } - sizingPolicyName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.sizingPolicy") - if err != nil && !strings.Contains(err.Error(), "key 'sizingPolicy' does not exist in input map") { - return nil, err - } - placementPolicyName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.placementPolicy") - if err != nil && !strings.Contains(err.Error(), "key 'placementPolicy' does not exist in input map") { - return nil, err - } - storageProfileName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.storageProfile") - if err != nil && !strings.Contains(err.Error(), "key 'storageProfile' does not exist in input map") { - return nil, err - } - diskSizeGiRaw, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.diskSize") - if err != nil { - return nil, err - } - diskSizeGi, err := strconv.Atoi(strings.ReplaceAll(diskSizeGiRaw, "Gi", "")) + name := traverseMapAndGet[string](yamlDocument, "metadata.name") + sizingPolicyName := traverseMapAndGet[string](yamlDocument, "spec.template.spec.sizingPolicy") + placementPolicyName := traverseMapAndGet[string](yamlDocument, "spec.template.spec.placementPolicy") + storageProfileName := traverseMapAndGet[string](yamlDocument, "spec.template.spec.storageProfile") + diskSizeGi, err := strconv.Atoi(strings.ReplaceAll(traverseMapAndGet[string](yamlDocument, "spec.template.spec.diskSize"), "Gi", "")) if err != nil { return nil, err } @@ -285,19 +257,11 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu result.ControlPlane.DiskSizeGi = diskSizeGi // We retrieve the Kubernetes Template OVA just once for the Control Plane because all YAML blocks share the same - catalogName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.catalog") - if err != nil { - return nil, err - } - catalog, err := rde.client.GetCatalogByName(result.capvcdType.Status.Capvcd.VcdProperties.Organizations[0].Name, catalogName) - if err != nil { - return nil, err - } - ovaName, err := traverseMapAndGet[string](yamlDocument, "spec.template.spec.template") + catalog, err := rde.client.GetCatalogByName(result.capvcdType.Status.Capvcd.VcdProperties.Organizations[0].Name, traverseMapAndGet[string](yamlDocument, "spec.template.spec.catalog")) if err != nil { return nil, err } - ova, err := catalog.GetVAppTemplateByName(ovaName) + ova, err := catalog.GetVAppTemplateByName(traverseMapAndGet[string](yamlDocument, "spec.template.spec.template")) if err != nil { return nil, err } @@ -328,41 +292,25 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu workerPools[name] = workerPool // Override the worker pool with the updated data } case "MachineDeployment": - name, err := traverseMapAndGet[string](yamlDocument, "metadata.name") - if err != nil { - return nil, err - } + name := traverseMapAndGet[string](yamlDocument, "metadata.name") // This is one Worker Pool. We need to check the map of worker pools, just in case we already saved the // other information from VCDMachineTemplate. if _, ok := workerPools[name]; !ok { workerPools[name] = CseWorkerPoolSettings{} } workerPool := workerPools[name] - replicas, err := traverseMapAndGet[float64](yamlDocument, "spec.replicas") - if err != nil { - return nil, err - } - workerPool.MachineCount = int(replicas) + workerPool.MachineCount = int(traverseMapAndGet[float64](yamlDocument, "spec.replicas")) workerPools[name] = workerPool // Override the worker pool with the updated data case "VCDCluster": - subnet, err := traverseMapAndGet[string](yamlDocument, "spec.loadBalancerConfigSpec.vipSubnet") - if err == nil { - result.VirtualIpSubnet = subnet // This is optional - } + result.VirtualIpSubnet = traverseMapAndGet[string](yamlDocument, "spec.loadBalancerConfigSpec.vipSubnet") case "Cluster": - cidrBlocks, err := traverseMapAndGet[[]interface{}](yamlDocument, "spec.clusterNetwork.pods.cidrBlocks") - if err != nil { - return nil, err - } + cidrBlocks := traverseMapAndGet[[]interface{}](yamlDocument, "spec.clusterNetwork.pods.cidrBlocks") if len(cidrBlocks) == 0 { return nil, fmt.Errorf("expected at least one 'spec.clusterNetwork.pods.cidrBlocks' item") } result.PodCidr = cidrBlocks[0].(string) - cidrBlocks, err = traverseMapAndGet[[]interface{}](yamlDocument, "spec.clusterNetwork.services.cidrBlocks") - if err != nil { - return nil, err - } + cidrBlocks = traverseMapAndGet[[]interface{}](yamlDocument, "spec.clusterNetwork.services.cidrBlocks") if len(cidrBlocks) == 0 { return nil, fmt.Errorf("expected at least one 'spec.clusterNetwork.services.cidrBlocks' item") } @@ -740,7 +688,8 @@ func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheck } result := &vcdKeConfig{} - // TODO: Check airgapped environments: https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.1.1a/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.1.1/GUID-F00BE796-B5F2-48F2-A012-546E2E694400.html + // We append /tkg as required, even in air-gapped environments: + // https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.2/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.2/GUID-B5C19221-2ECA-4DCD-8EA1-8E391F6217C1.html result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"]) if isNodeHealthCheckActive { @@ -765,7 +714,7 @@ func getCseTemplate(cseVersion semver.Version, templateName string) (string, err return "", err } if cseVersion.LessThan(minimumVersion) { - return "", fmt.Errorf("the Container Service version '%s' is not supported", minimumVersion.String()) + return "", fmt.Errorf("the Container Service minimum version is '%s'", minimumVersion.String()) } versionSegments := cseVersion.Segments() fullTemplatePath := fmt.Sprintf("cse/%d.%d/%s.tmpl", versionSegments[0], versionSegments[1], templateName) diff --git a/govcd/cse_util_test.go b/govcd/cse_util_test.go deleted file mode 100644 index daa5f2889..000000000 --- a/govcd/cse_util_test.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build functional || openapi || cse || ALL - -/* - * Copyright 2024 VMware, Inc. All rights reserved. Licensed under the Apache v2 License. - */ - -package govcd diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index 17f8c96d6..11b5c3fc5 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -7,6 +7,7 @@ package govcd import ( + semver "github.com/hashicorp/go-version" "reflect" "testing" ) @@ -66,3 +67,43 @@ func Test_getTkgVersionBundleFromVAppTemplateName(t *testing.T) { }) } } + +func Test_getCseTemplate(t *testing.T) { + v40, err := semver.NewVersion("4.0") + if err != nil { + t.Fatalf("could not create 4.0 version object") + } + v41, err := semver.NewVersion("4.1") + if err != nil { + t.Fatalf("could not create 4.1 version object") + } + v410, err := semver.NewVersion("4.1.0") + if err != nil { + t.Fatalf("could not create 4.1.0 version object") + } + v411, err := semver.NewVersion("4.1.1") + if err != nil { + t.Fatalf("could not create 4.1.1 version object") + } + + tmpl41, err := getCseTemplate(*v41, "rde") + if err != nil { + t.Fatalf("%s", err) + } + tmpl410, err := getCseTemplate(*v410, "rde") + if err != nil { + t.Fatalf("%s", err) + } + tmpl411, err := getCseTemplate(*v411, "rde") + if err != nil { + t.Fatalf("%s", err) + } + if tmpl41 != tmpl410 || tmpl41 != tmpl411 || tmpl410 != tmpl411 { + t.Fatalf("templates should be the same:\n4.1: %s\n4.1.0: %s\n4.1.1: %s", tmpl41, tmpl410, tmpl411) + } + + _, err = getCseTemplate(*v40, "rde") + if err == nil && err.Error() != "the Container Service minimum version is '4.1.0'" { + t.Fatalf("expected an error but got %s", err) + } +} diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 8fc87710b..183916bb4 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -89,42 +89,42 @@ func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]interface{}, k for _, d := range yamlDocuments { switch d["kind"] { case "VCDMachineTemplate": - _, err := traverseMapAndGet[string](d, "spec.template.spec.template") - if err != nil { - return fmt.Errorf("incorrect YAML: %s", err) + ok := traverseMapAndGet[string](d, "spec.template.spec.template") != "" + if !ok { + return fmt.Errorf("the VCDMachineTemplate 'spec.template.spec.template' field is missing") } d["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{})["template"] = kubernetesTemplateOvaName case "MachineDeployment": - _, err := traverseMapAndGet[string](d, "spec.template.spec.version") - if err != nil { - return fmt.Errorf("incorrect YAML: %s", err) + ok := traverseMapAndGet[string](d, "spec.template.spec.version") != "" + if !ok { + return fmt.Errorf("the MachineDeployment 'spec.template.spec.version' field is missing") } d["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{})["version"] = tkgBundle.KubernetesVersion case "Cluster": - _, err := traverseMapAndGet[string](d, "metadata.annotations.TKGVERSION") - if err != nil { - return fmt.Errorf("incorrect YAML: %s", err) + ok := traverseMapAndGet[string](d, "metadata.annotations.TKGVERSION") != "" + if !ok { + return fmt.Errorf("the Cluster 'metadata.annotations.TKGVERSION' field is missing") } d["metadata"].(map[string]interface{})["annotations"].(map[string]interface{})["TKGVERSION"] = tkgBundle.TkgVersion - _, err = traverseMapAndGet[string](d, "metadata.labels.tanzuKubernetesRelease") - if err != nil { - return fmt.Errorf("incorrect YAML: %s", err) + ok = traverseMapAndGet[string](d, "metadata.labels.tanzuKubernetesRelease") != "" + if !ok { + return fmt.Errorf("the Cluster 'metadata.labels.tanzuKubernetesRelease' field is missing") } d["metadata"].(map[string]interface{})["labels"].(map[string]interface{})["tanzuKubernetesRelease"] = tkgBundle.TkrVersion case "KubeadmControlPlane": - _, err := traverseMapAndGet[string](d, "spec.version") - if err != nil { - return fmt.Errorf("incorrect YAML: %s", err) + ok := traverseMapAndGet[string](d, "spec.version") != "" + if !ok { + return fmt.Errorf("the KubeadmControlPlane 'spec.version' field is missing") } d["spec"].(map[string]interface{})["version"] = tkgBundle.KubernetesVersion - _, err = traverseMapAndGet[string](d, "spec.kubeadmConfigSpec.clusterConfiguration.dns.imageTag") - if err != nil { - return fmt.Errorf("incorrect YAML: %s", err) + ok = traverseMapAndGet[string](d, "spec.kubeadmConfigSpec.clusterConfiguration.dns.imageTag") != "" + if !ok { + return fmt.Errorf("the KubeadmControlPlane 'spec.kubeadmConfigSpec.clusterConfiguration.dns.imageTag' field is missing") } d["spec"].(map[string]interface{})["kubeadmConfigSpec"].(map[string]interface{})["clusterConfiguration"].(map[string]interface{})["dns"].(map[string]interface{})["imageTag"] = tkgBundle.CoreDnsVersion - _, err = traverseMapAndGet[string](d, "spec.kubeadmConfigSpec.clusterConfiguration.etcd.local.imageTag") - if err != nil { - return fmt.Errorf("incorrect YAML: %s", err) + ok = traverseMapAndGet[string](d, "spec.kubeadmConfigSpec.clusterConfiguration.etcd.local.imageTag") != "" + if !ok { + return fmt.Errorf("the KubeadmControlPlane 'spec.kubeadmConfigSpec.clusterConfiguration.etcd.local.imageTag' field is missing") } d["spec"].(map[string]interface{})["kubeadmConfigSpec"].(map[string]interface{})["clusterConfiguration"].(map[string]interface{})["etcd"].(map[string]interface{})["local"].(map[string]interface{})["imageTag"] = tkgBundle.EtcdVersion } @@ -143,10 +143,6 @@ func cseUpdateControlPlaneInYaml(yamlDocuments []map[string]interface{}, input C if d["kind"] != "KubeadmControlPlane" { continue } - _, err := traverseMapAndGet[float64](d, "spec.replicas") - if err != nil { - return fmt.Errorf("incorrect YAML: %s", err) - } d["spec"].(map[string]interface{})["replicas"] = float64(input.MachineCount) // As it was originally unmarshalled as a float64 updated = true } @@ -165,9 +161,9 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]interface{}, workerPo continue } - workerPoolName, err := traverseMapAndGet[string](d, "metadata.name") - if err != nil { - return fmt.Errorf("incorrect YAML: %s", err) + workerPoolName := traverseMapAndGet[string](d, "metadata.name") + if workerPoolName == "" { + return fmt.Errorf("the MachineDeployment 'metadata.name' field is empty") } workerPoolToUpdate := "" @@ -185,11 +181,6 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]interface{}, workerPo return fmt.Errorf("incorrect machine count for worker pool %s: %d. Should be at least 0", workerPoolToUpdate, workerPools[workerPoolToUpdate].MachineCount) } - _, err = traverseMapAndGet[float64](d, "spec.replicas") - if err != nil { - return fmt.Errorf("incorrect YAML: %s", err) - } - d["spec"].(map[string]interface{})["replicas"] = float64(workerPools[workerPoolToUpdate].MachineCount) // As it was originally unmarshalled as a float64 updated++ } @@ -319,18 +310,18 @@ func unmarshalMultipleYamlDocuments(yamlDocuments string) ([]map[string]interfac // traverseMapAndGet traverses the input interface{}, which should be a map of maps, by following the path specified as // "keyA.keyB.keyC.keyD", doing something similar to, visually speaking, map["keyA"]["keyB"]["keyC"]["keyD"], or in other words, // it goes inside every inner map iteratively, until the given path is finished. -// The final value, "keyD" in the same example, should be of any type T. -func traverseMapAndGet[T any](input interface{}, path string) (T, error) { +// If the path doesn't lead to any value, or if the value is nil, or there is any other issue, returns the "zero" value of T. +func traverseMapAndGet[T any](input interface{}, path string) T { var nothing T if input == nil { - return nothing, fmt.Errorf("the input is nil") + return nothing } inputMap, ok := input.(map[string]interface{}) if !ok { - return nothing, fmt.Errorf("the input is a %T, not a map[string]interface{}", input) + return nothing } if len(inputMap) == 0 { - return nothing, fmt.Errorf("the map is empty") + return nothing } pathUnits := strings.Split(path, ".") completed := false @@ -340,12 +331,12 @@ func traverseMapAndGet[T any](input interface{}, path string) (T, error) { subPath := pathUnits[i] traversed, ok := inputMap[subPath] if !ok { - return nothing, fmt.Errorf("key '%s' does not exist in input map", subPath) + return nothing } if i < len(pathUnits)-1 { traversedMap, ok := traversed.(map[string]interface{}) if !ok { - return nothing, fmt.Errorf("key '%s' is a %T, not a map[string]interface{}, but there are still %d paths to explore", subPath, traversed, len(pathUnits)-(i+1)) + return nothing } inputMap = traversedMap } else { @@ -356,7 +347,7 @@ func traverseMapAndGet[T any](input interface{}, path string) (T, error) { } resultTyped, ok := result.(T) if !ok { - return nothing, fmt.Errorf("could not convert obtained type %T to requested %T", result, nothing) + return nothing } - return resultTyped, nil + return resultTyped } diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 937892db3..5a825b346 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -30,8 +30,8 @@ func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { continue } - oldOvaName, err = traverseMapAndGet[string](document, "spec.template.spec.template") - if err != nil { + oldOvaName = traverseMapAndGet[string](document, "spec.template.spec.template") + if oldOvaName == "" { t.Fatalf("expected to find spec.template.spec.template in %v but got an error: %s", document, err) } break @@ -101,17 +101,13 @@ func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { continue } - workerPoolName, err := traverseMapAndGet[string](document, "metadata.name") - if err != nil { + workerPoolName := traverseMapAndGet[string](document, "metadata.name") + if workerPoolName == "" { t.Fatalf("incorrect CAPI YAML: %s", err) } - oldReplicas, err := traverseMapAndGet[float64](document, "spec.replicas") - if err != nil { - t.Fatalf("incorrect CAPI YAML: %s", err) - } oldNodePools[workerPoolName] = CseWorkerPoolUpdateInput{ - MachineCount: int(oldReplicas), + MachineCount: int(traverseMapAndGet[float64](document, "spec.replicas")), } } if len(oldNodePools) == 0 { @@ -137,11 +133,8 @@ func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { continue } - retrievedReplicas, err := traverseMapAndGet[float64](document, "spec.replicas") - if err != nil { - t.Fatalf("incorrect CAPI YAML: %s", err) - } - if retrievedReplicas != float64(newReplicas) { + retrievedReplicas := traverseMapAndGet[float64](document, "spec.replicas") + if traverseMapAndGet[float64](document, "spec.replicas") != float64(newReplicas) { t.Fatalf("expected %d replicas but got %0.f", newReplicas, retrievedReplicas) } } @@ -223,10 +216,7 @@ func Test_cseAddWorkerPoolsInYaml(t *testing.T) { continue } - name, err := traverseMapAndGet[string](document, "metadata.name") - if err != nil { - t.Fatalf("incorrect CAPI YAML: %s", err) - } + name := traverseMapAndGet[string](document, "metadata.name") if name == "new-pool" { newPool = document } @@ -238,10 +228,7 @@ func Test_cseAddWorkerPoolsInYaml(t *testing.T) { if poolCount != newPoolCount-1 { t.Fatalf("should have one extra Worker Pool") } - replicas, err := traverseMapAndGet[float64](newPool, "spec.replicas") - if err != nil { - t.Fatalf("incorrect CAPI YAML: %s", err) - } + replicas := traverseMapAndGet[float64](newPool, "spec.replicas") if replicas != 35 { t.Fatalf("incorrect replicas: %.f", replicas) } @@ -266,12 +253,8 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { continue } - oldReplicas, err := traverseMapAndGet[float64](document, "spec.replicas") - if err != nil { - t.Fatalf("incorrect CAPI YAML: %s", err) - } oldControlPlane = CseControlPlaneUpdateInput{ - MachineCount: int(oldReplicas), + MachineCount: int(traverseMapAndGet[float64](document, "spec.replicas")), } } if reflect.DeepEqual(oldControlPlane, CseWorkerPoolUpdateInput{}) { @@ -294,10 +277,7 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { continue } - retrievedReplicas, err := traverseMapAndGet[float64](document, "spec.replicas") - if err != nil { - t.Fatalf("incorrect CAPI YAML: %s", err) - } + retrievedReplicas := traverseMapAndGet[float64](document, "spec.replicas") if retrievedReplicas != float64(newReplicas) { t.Fatalf("expected %d replicas but got %0.f", newReplicas, retrievedReplicas) } @@ -331,10 +311,7 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { if doc["kind"] != "Cluster" { continue } - clusterName, err = traverseMapAndGet[string](doc, "metadata.name") - if err != nil { - t.Fatalf("incorrect CAPI YAML: %s", err) - } + clusterName = traverseMapAndGet[string](doc, "metadata.name") } if clusterName == "" { t.Fatal("could not find the cluster name in the CAPI YAML test file") @@ -375,17 +352,11 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { if document["kind"] != "MachineHealthCheck" { continue } - maxUnhealthy, err := traverseMapAndGet[string](document, "spec.maxUnhealthy") - if err != nil { - t.Fatalf("%s", err) - } + maxUnhealthy := traverseMapAndGet[string](document, "spec.maxUnhealthy") if maxUnhealthy != "12%" { t.Fatalf("expected a 'spec.maxUnhealthy' = 12%%, but got %s", maxUnhealthy) } - nodeStartupTimeout, err := traverseMapAndGet[string](document, "spec.nodeStartupTimeout") - if err != nil { - t.Fatalf("%s", err) - } + nodeStartupTimeout := traverseMapAndGet[string](document, "spec.nodeStartupTimeout") if nodeStartupTimeout != "34s" { t.Fatalf("expected a 'spec.nodeStartupTimeout' = 34s, but got %s", nodeStartupTimeout) } @@ -499,36 +470,38 @@ func Test_marshalMultplieYamlDocuments(t *testing.T) { // Test_traverseMapAndGet tests traverseMapAndGet function func Test_traverseMapAndGet(t *testing.T) { type args struct { - input any + input interface{} path string } tests := []struct { name string args args wantType string - want any - wantErr string + want interface{} }{ { name: "input is nil", args: args{ input: nil, }, - wantErr: "the input is nil", + wantType: "string", + want: "", }, { name: "input is not a map", args: args{ input: "error", }, - wantErr: "the input is a string, not a map[string]interface{}", + wantType: "string", + want: "", }, { name: "map is empty", args: args{ input: map[string]interface{}{}, }, - wantErr: "the map is empty", + wantType: "float64", + want: float64(0), }, { name: "map does not have key", @@ -538,7 +511,8 @@ func Test_traverseMapAndGet(t *testing.T) { }, path: "keyB", }, - wantErr: "key 'keyB' does not exist in input map", + wantType: "string", + want: "", }, { name: "map has a single simple key", @@ -593,7 +567,8 @@ func Test_traverseMapAndGet(t *testing.T) { }, path: "keyA.keyB.keyC.keyD", }, - wantErr: "key 'keyC' is a string, not a map, but there are still 1 paths to explore", + wantType: "string", + want: "", }, { name: "obtained value does not correspond to the desired type", @@ -608,27 +583,22 @@ func Test_traverseMapAndGet(t *testing.T) { path: "keyA.keyB.keyC", }, wantType: "string", - wantErr: "could not convert obtained type map[string]interface {} to requested string", + want: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var got any - var err error + var got interface{} if tt.wantType == "string" { - got, err = traverseMapAndGet[string](tt.args.input, tt.args.path) + got = traverseMapAndGet[string](tt.args.input, tt.args.path) } else if tt.wantType == "map" { - got, err = traverseMapAndGet[map[string]interface{}](tt.args.input, tt.args.path) + got = traverseMapAndGet[map[string]interface{}](tt.args.input, tt.args.path) + } else if tt.wantType == "float64" { + got = traverseMapAndGet[float64](tt.args.input, tt.args.path) } else { t.Fatalf("wantType type not used in this test") } - if err != nil { - if tt.wantErr != err.Error() { - t.Errorf("traverseMapAndGet() error = %v, wantErr = %v", err, tt.wantErr) - } - return - } if !reflect.DeepEqual(got, tt.want) { t.Errorf("traverseMapAndGet() got = %v, want %v", got, tt.want) } From 5cead14bdf2c92a1064f05b89e7e84bddfbff24b Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 9 Feb 2024 12:13:10 +0100 Subject: [PATCH 028/115] check Signed-off-by: abarreiro --- types/v56/cse.go | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/types/v56/cse.go b/types/v56/cse.go index 42424b005..790954fb4 100644 --- a/types/v56/cse.go +++ b/types/v56/cse.go @@ -34,6 +34,15 @@ type Capvcd struct { DetailedEvent string `json:"Detailed Event,omitempty"` } `json:"additionalDetails,omitempty"` } `json:"eventSet,omitempty"` + ErrorSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` + AdditionalDetails struct { + DetailedError string `json:"Detailed Error,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"errorSet,omitempty"` } `json:"cpi,omitempty"` Csi struct { Name string `json:"name,omitempty"` @@ -45,6 +54,15 @@ type Capvcd struct { DetailedDescription string `json:"Detailed Description,omitempty"` } `json:"additionalDetails,omitempty"` } `json:"eventSet,omitempty"` + ErrorSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` + AdditionalDetails struct { + DetailedError string `json:"Detailed Error,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"errorSet,omitempty"` } `json:"csi,omitempty"` VcdKe struct { State string `json:"state,omitempty"` @@ -94,15 +112,16 @@ type Capvcd struct { TkgVersion string `json:"tkgVersion,omitempty"` KubernetesVersion string `json:"kubernetesVersion,omitempty"` } `json:"current,omitempty"` + Target struct { + TkgVersion string `json:"tkgVersion,omitempty"` + KubernetesVersion string `json:"kubernetesVersion,omitempty"` + } `json:"target,omitempty"` } `json:"upgrade,omitempty"` EventSet []struct { - Name string `json:"name,omitempty"` - OccurredAt time.Time `json:"occurredAt,omitempty"` - VcdResourceId string `json:"vcdResourceId,omitempty"` - VcdResourceName string `json:"vcdResourceName,omitempty"` - AdditionalDetails struct { - Event string `json:"event,omitempty"` - } `json:"additionalDetails,omitempty"` + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` } `json:"eventSet,omitempty"` ErrorSet []struct { Name string `json:"name,omitempty"` @@ -180,6 +199,15 @@ type Capvcd struct { Event string `json:"event,omitempty"` } `json:"additionalDetails,omitempty"` } `json:"eventSet,omitempty"` + ErrorSet []struct { + Name string `json:"name,omitempty"` + OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` + AdditionalDetails struct { + DetailedError string `json:"Detailed Error,omitempty"` + } `json:"additionalDetails,omitempty"` + } `json:"errorSet,omitempty"` } `json:"projector,omitempty"` } `json:"status,omitempty"` Metadata struct { From 5e3728a00b04f27c16352bd27f2f387524133f75 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 9 Feb 2024 12:28:30 +0100 Subject: [PATCH 029/115] Add upgrade constraints Signed-off-by: abarreiro --- govcd/cse_test.go | 53 +++++++++++++++++++++++++++++++++++++++++++---- govcd/cse_type.go | 1 + govcd/cse_util.go | 1 + govcd/cse_yaml.go | 23 ++++++++++---------- 4 files changed, 63 insertions(+), 15 deletions(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 6005ab9f7..fd67e1ba4 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -109,14 +109,34 @@ func (vcd *TestVCD) Test_Cse(check *C) { cluster, err := org.CseCreateKubernetesCluster(clusterSettings, 0) check.Assert(err, IsNil) + check.Assert(cluster.CseVersion.String(), Equals, cseVersion.String()) + check.Assert(cluster.Name, Equals, clusterSettings.Name) + check.Assert(cluster.OrganizationId, Equals, clusterSettings.OrganizationId) + check.Assert(cluster.VdcId, Equals, clusterSettings.VdcId) + check.Assert(cluster.NetworkId, Equals, clusterSettings.NetworkId) + check.Assert(cluster.KubernetesTemplateOvaId, Equals, clusterSettings.KubernetesTemplateOvaId) + check.Assert(cluster.ControlPlane, DeepEquals, clusterSettings.ControlPlane) + check.Assert(cluster.WorkerPools, DeepEquals, clusterSettings.WorkerPools) + check.Assert(cluster.DefaultStorageClass, NotNil) + check.Assert(*cluster.DefaultStorageClass, DeepEquals, *clusterSettings.DefaultStorageClass) + check.Assert(cluster.Owner, Equals, clusterSettings.Owner) + check.Assert(cluster.ApiToken, Not(Equals), clusterSettings.ApiToken) + check.Assert(cluster.ApiToken, Equals, "******") // This one can't be recovered + check.Assert(cluster.NodeHealthCheck, Equals, clusterSettings.NodeHealthCheck) + check.Assert(cluster.PodCidr, Equals, clusterSettings.PodCidr) + check.Assert(cluster.ServiceCidr, Equals, clusterSettings.ServiceCidr) + check.Assert(cluster.SshPublicKey, Equals, clusterSettings.SshPublicKey) + check.Assert(cluster.VirtualIpSubnet, Equals, clusterSettings.VirtualIpSubnet) + check.Assert(cluster.AutoRepairOnErrors, Equals, clusterSettings.AutoRepairOnErrors) + check.Assert(cluster.VirtualIpSubnet, Equals, clusterSettings.VirtualIpSubnet) check.Assert(true, Equals, strings.Contains(cluster.ID, "urn:vcloud:entity:vmware:capvcdCluster:")) check.Assert(cluster.Etag, Not(Equals), "") - check.Assert(cluster.CseClusterSettings, DeepEquals, clusterSettings) check.Assert(cluster.KubernetesVersion, Equals, tkgBundle.KubernetesVersion) check.Assert(cluster.TkgVersion, Equals, tkgBundle.TkgVersion) check.Assert(cluster.CapvcdVersion, Not(Equals), "") check.Assert(cluster.CpiVersion, Not(Equals), "") check.Assert(cluster.CsiVersion, Not(Equals), "") + check.Assert(cluster.Upgradeable, Equals, true) check.Assert(len(cluster.ClusterResourceSetBindings), Not(Equals), 0) check.Assert(cluster.State, Equals, "provisioned") check.Assert(len(cluster.Events), Not(Equals), 0) @@ -133,11 +153,36 @@ func (vcd *TestVCD) Test_Cse(check *C) { clusterGet, err := org.CseGetKubernetesClusterById(cluster.ID) check.Assert(err, IsNil) - check.Assert(cluster.ID, Equals, clusterGet.ID) + check.Assert(cluster.CseVersion.String(), Equals, clusterGet.CseVersion.String()) check.Assert(cluster.Name, Equals, clusterGet.Name) + check.Assert(cluster.OrganizationId, Equals, clusterGet.OrganizationId) + check.Assert(cluster.VdcId, Equals, clusterGet.VdcId) + check.Assert(cluster.NetworkId, Equals, clusterGet.NetworkId) + check.Assert(cluster.KubernetesTemplateOvaId, Equals, clusterGet.KubernetesTemplateOvaId) + check.Assert(cluster.ControlPlane, DeepEquals, clusterGet.ControlPlane) + check.Assert(cluster.WorkerPools, DeepEquals, clusterGet.WorkerPools) + check.Assert(cluster.DefaultStorageClass, NotNil) + check.Assert(*cluster.DefaultStorageClass, DeepEquals, *clusterGet.DefaultStorageClass) check.Assert(cluster.Owner, Equals, clusterGet.Owner) - check.Assert(cluster.capvcdType.Metadata, DeepEquals, clusterGet.capvcdType.Metadata) - check.Assert(cluster.capvcdType.Spec.VcdKe, DeepEquals, clusterGet.capvcdType.Spec.VcdKe) + check.Assert(cluster.ApiToken, Not(Equals), clusterGet.ApiToken) + check.Assert(clusterGet.ApiToken, Equals, "******") // This one can't be recovered + check.Assert(cluster.NodeHealthCheck, Equals, clusterGet.NodeHealthCheck) + check.Assert(cluster.PodCidr, Equals, clusterGet.PodCidr) + check.Assert(cluster.ServiceCidr, Equals, clusterGet.ServiceCidr) + check.Assert(cluster.SshPublicKey, Equals, clusterGet.SshPublicKey) + check.Assert(cluster.VirtualIpSubnet, Equals, clusterGet.VirtualIpSubnet) + check.Assert(cluster.AutoRepairOnErrors, Equals, clusterGet.AutoRepairOnErrors) + check.Assert(cluster.VirtualIpSubnet, Equals, clusterGet.VirtualIpSubnet) + check.Assert(cluster.ID, Equals, clusterGet.ID) + check.Assert(clusterGet.Etag, Not(Equals), "") + check.Assert(cluster.KubernetesVersion, Equals, clusterGet.KubernetesVersion) + check.Assert(cluster.TkgVersion.String(), Equals, clusterGet.TkgVersion.String()) + check.Assert(cluster.CapvcdVersion.String(), Equals, clusterGet.CapvcdVersion.String()) + check.Assert(cluster.ClusterResourceSetBindings, DeepEquals, clusterGet.ClusterResourceSetBindings) + check.Assert(cluster.CpiVersion.String(), Equals, clusterGet.CpiVersion.String()) + check.Assert(cluster.CsiVersion.String(), Equals, clusterGet.CsiVersion.String()) + check.Assert(cluster.Upgradeable, Equals, clusterGet.Upgradeable) + check.Assert(cluster.State, Equals, clusterGet.State) // Update worker pool from 1 node to 2 // Pre-check. This should be 1, as it was created with just 1 pool diff --git a/govcd/cse_type.go b/govcd/cse_type.go index ebd91e228..538070957 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -19,6 +19,7 @@ type CseKubernetesCluster struct { CpiVersion semver.Version CsiVersion semver.Version State string + Upgradeable bool Events []CseClusterEvent client *Client diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 4216f0b38..238610085 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -63,6 +63,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu ID: rde.DefinedEntity.ID, Etag: rde.Etag, ClusterResourceSetBindings: make([]string, len(capvcd.Status.Capvcd.ClusterResourceSetBindings)), + Upgradeable: capvcd.Status.Capvcd.Upgrade.Target.KubernetesVersion != "" && capvcd.Status.Capvcd.Upgrade.Target.TkgVersion != "", State: capvcd.Status.VcdKe.State, client: rde.client, capvcdType: capvcd, diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 183916bb4..05a01c2e0 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -32,25 +32,26 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) // as well. // So in this special case this "optimization" would optimize nothing. The same happens with other YAML values. if input.KubernetesTemplateOvaId != nil { + if !cluster.Upgradeable { + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the cluster is not upgradeable") + } vAppTemplate, err := getVAppTemplateById(cluster.client, *input.KubernetesTemplateOvaId) if err != nil { return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", *input.KubernetesTemplateOvaId, err) } - err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate.Name) + // Check the versions of the selected OVA before upgrading + versions, err := getTkgVersionBundleFromVAppTemplateName(vAppTemplate.VAppTemplate.Name) if err != nil { - return cluster.capvcdType.Spec.CapiYaml, err + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the TKG versions of OVA '%s': %s", vAppTemplate.VAppTemplate.Name, err) } - } - if input.ControlPlane != nil { - err := cseUpdateControlPlaneInYaml(yamlDocs, *input.ControlPlane) - if err != nil { - return cluster.capvcdType.Spec.CapiYaml, err + if versions.TkgVersion < cluster.capvcdType.Status.Capvcd.Upgrade.Target.TkgVersion { + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the new one '%s' has an older TKG version", vAppTemplate.VAppTemplate.Name) } - } - - if input.WorkerPools != nil { - err := cseUpdateWorkerPoolsInYaml(yamlDocs, *input.WorkerPools) + if versions.KubernetesVersion < cluster.capvcdType.Status.Capvcd.Upgrade.Target.KubernetesVersion { + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the new one '%s' has an older TKG version", vAppTemplate.VAppTemplate.Name) + } + err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate.Name) if err != nil { return cluster.capvcdType.Spec.CapiYaml, err } From e3dcf72fab653a5936c2b70b55616779ab07f0f0 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 9 Feb 2024 13:25:47 +0100 Subject: [PATCH 030/115] Fix getTkgVersionBundleFromVAppTemplate Signed-off-by: abarreiro --- govcd/cse.go | 9 +++- govcd/cse_test.go | 2 +- govcd/cse_util.go | 50 ++++++++++---------- govcd/cse_util_unit_test.go | 93 ++++++++++++++++++++++++++++--------- govcd/cse_yaml.go | 13 +++--- govcd/cse_yaml_unit_test.go | 60 +++++++++++++++--------- types/v56/types.go | 1 + 7 files changed, 150 insertions(+), 78 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 6fb313e7c..bdf4fc240 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -150,9 +150,14 @@ func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpd }, refresh) } -// ChangeKubernetesTemplateOva executes an update on the receiver cluster to change the Kubernetes template of the cluster. +// GetSupportedUpgrades gets a list of Kubernetes Template OVA IDs that can be used for a cluster upgrade +func (cluster *CseKubernetesCluster) GetSupportedUpgrades() ([]string, error) { + return nil, nil +} + +// UpgradeCluster executes an update on the receiver cluster to change the Kubernetes template of the cluster. // If refresh=true, it retrieves the latest state of the cluster from VCD before updating. -func (cluster *CseKubernetesCluster) ChangeKubernetesTemplateOva(kubernetesTemplateOvaId string, refresh bool) error { +func (cluster *CseKubernetesCluster) UpgradeCluster(kubernetesTemplateOvaId string, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ KubernetesTemplateOvaId: &kubernetesTemplateOvaId, }, refresh) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index fd67e1ba4..6173f1611 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -43,7 +43,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { ova, err := catalog.GetVAppTemplateByName(vcd.config.Cse.OvaName) check.Assert(err, IsNil) - tkgBundle, err := getTkgVersionBundleFromVAppTemplateName(ova.VAppTemplate.Name) + tkgBundle, err := getTkgVersionBundleFromVAppTemplate(ova.VAppTemplate) check.Assert(err, IsNil) vdc, err := org.GetVDCByName(vcd.config.Cse.TenantVdc, false) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 238610085..4a5576961 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -488,7 +488,7 @@ func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClus } output.KubernetesTemplateOvaName = vAppTemplate.VAppTemplate.Name - tkgVersions, err := getTkgVersionBundleFromVAppTemplateName(vAppTemplate.VAppTemplate.Name) + tkgVersions, err := getTkgVersionBundleFromVAppTemplate(vAppTemplate.VAppTemplate) if err != nil { return nil, fmt.Errorf("could not retrieve the required information from the Kubernetes Template OVA: %s", err) } @@ -619,24 +619,29 @@ func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClus return output, nil } -// getTkgVersionBundleFromVAppTemplateName returns a tkgVersionBundle with the details of -// all the Kubernetes cluster components versions given a valid Kubernetes Template OVA name. +// getTkgVersionBundleFromVAppTemplate returns a tkgVersionBundle with the details of +// all the Kubernetes cluster components versions given a valid Kubernetes Template OVA. // If it is not a valid Kubernetes Template OVA, returns an error. -func getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName string) (tkgVersionBundle, error) { +func getTkgVersionBundleFromVAppTemplate(template *types.VAppTemplate) (tkgVersionBundle, error) { result := tkgVersionBundle{} - if strings.TrimSpace(kubernetesTemplateOvaName) == "" { - return result, fmt.Errorf("the Kubernetes Template OVA cannot be empty") + if template == nil { + return result, fmt.Errorf("the Kubernetes Template OVA is nil") } - - if strings.Contains(kubernetesTemplateOvaName, "photon") { - return result, fmt.Errorf("the Kubernetes Template OVA '%s' uses Photon, and it is not supported", kubernetesTemplateOvaName) + if template.Children == nil || len(template.Children.VM) == 0 { + return result, fmt.Errorf("the Kubernetes Template OVA '%s' doesn't have any child VM", template.Name) } - - cutPosition := strings.LastIndex(kubernetesTemplateOvaName, "kube-") - if cutPosition < 0 { - return result, fmt.Errorf("the OVA '%s' is not a Kubernetes template OVA", kubernetesTemplateOvaName) + if template.Children.VM[0].ProductSection == nil { + return result, fmt.Errorf("the Product section of the Kubernetes Template OVA '%s' is empty, can't proceed", template.Name) + } + id := "" + for _, prop := range template.Children.VM[0].ProductSection.Property { + if prop != nil && prop.Key == "VERSION" && prop.Value != nil { + id = prop.Value.Value + } + } + if id == "" { + return result, fmt.Errorf("could not find any Version inside the Kubernetes Template OVA '%s' Product section properties", template.Name) } - parsedOvaName := strings.ReplaceAll(kubernetesTemplateOvaName, ".ova", "")[cutPosition+len("kube-"):] tkgVersionsMap := "cse/tkg_versions.json" cseTkgVersionsJson, err := cseFiles.ReadFile(tkgVersionsMap) @@ -649,20 +654,15 @@ func getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName string) ( if err != nil { return result, fmt.Errorf("failed unmarshaling %s: %s", tkgVersionsMap, err) } - versionMap, ok := versionsMap[parsedOvaName] + versionMap, ok := versionsMap[id] if !ok { - return result, fmt.Errorf("the Kubernetes Template OVA '%s' is not supported", kubernetesTemplateOvaName) - } - - // This check should not be needed unless the tkgVersionsMap JSON is deliberately bad constructed. - ovaParts := strings.Split(parsedOvaName, "-") - if len(ovaParts) != 3 { - return result, fmt.Errorf("unexpected error parsing the Kubernetes Template OVA name '%s',"+ - "it doesn't follow the original naming convention (e.g: ubuntu-2004-kube-v1.24.11+vmware.1-tkg.1-2ccb2a001f8bd8f15f1bfbc811071830)", kubernetesTemplateOvaName) + return result, fmt.Errorf("the Kubernetes Template OVA '%s' is not supported", template.Name) } - result.KubernetesVersion = ovaParts[0] - result.TkrVersion = strings.ReplaceAll(ovaParts[0], "+", "---") + "-" + ovaParts[1] + // We don't need to check the Split result because the map checking above guarantees that the ID is well-formed. + idParts := strings.Split(id, "-") + result.KubernetesVersion = idParts[0] + result.TkrVersion = strings.ReplaceAll(idParts[0], "+", "---") + "-" + idParts[1] result.TkgVersion = versionMap.(map[string]interface{})["tkg"].(string) result.EtcdVersion = versionMap.(map[string]interface{})["etcd"].(string) result.CoreDnsVersion = versionMap.(map[string]interface{})["coreDns"].(string) diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index 11b5c3fc5..4cc21ca66 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -8,41 +8,88 @@ package govcd import ( semver "github.com/hashicorp/go-version" + "github.com/vmware/go-vcloud-director/v2/types/v56" "reflect" "testing" ) -// Test_getTkgVersionBundleFromVAppTemplateName tests getTkgVersionBundleFromVAppTemplateName function -func Test_getTkgVersionBundleFromVAppTemplateName(t *testing.T) { +// Test_getTkgVersionBundleFromVAppTemplate tests getTkgVersionBundleFromVAppTemplate function +func Test_getTkgVersionBundleFromVAppTemplate(t *testing.T) { tests := []struct { - name string - kubernetesTemplateOvaName string - want tkgVersionBundle - wantErr string + name string + kubernetesTemplateOva *types.VAppTemplate + want tkgVersionBundle + wantErr string }{ { - name: "input is empty", - kubernetesTemplateOvaName: "", - wantErr: "the Kubernetes Template OVA cannot be empty", + name: "ova is nil", + kubernetesTemplateOva: nil, + wantErr: "the Kubernetes Template OVA is nil", }, { - name: "input is Photon OVA", - kubernetesTemplateOvaName: "photon-2004-kube-v9.99.9+vmware.9-tkg.9-aaaaa.ova", - wantErr: "the Kubernetes Template OVA 'photon-2004-kube-v9.99.9+vmware.9-tkg.9-aaaaa.ova' uses Photon, and it is not supported", + name: "ova without children", + kubernetesTemplateOva: &types.VAppTemplate{ + Name: "dummy", + Children: nil, + }, + wantErr: "the Kubernetes Template OVA 'dummy' doesn't have any child VM", }, { - name: "input is not a Kubernetes OVA", - kubernetesTemplateOvaName: "random-ova.ova", - wantErr: "the OVA 'random-ova.ova' is not a Kubernetes template OVA", + name: "ova with nil children", + kubernetesTemplateOva: &types.VAppTemplate{ + Name: "dummy", + Children: &types.VAppTemplateChildren{VM: nil}, + }, + wantErr: "the Kubernetes Template OVA 'dummy' doesn't have any child VM", }, { - name: "input is not supported", - kubernetesTemplateOvaName: "ubuntu-2004-kube-v9.99.9+vmware.9-tkg.9-99999999999999999999999999999999.ova", - wantErr: "the Kubernetes Template OVA 'ubuntu-2004-kube-v9.99.9+vmware.9-tkg.9-99999999999999999999999999999999.ova' is not supported", + name: "ova with nil product section", + kubernetesTemplateOva: &types.VAppTemplate{ + Name: "dummy", + Children: &types.VAppTemplateChildren{VM: []*types.VAppTemplate{ + { + ProductSection: nil, + }, + }}, + }, + wantErr: "the Product section of the Kubernetes Template OVA 'dummy' is empty, can't proceed", }, { - name: "correct OVA", - kubernetesTemplateOvaName: "ubuntu-2004-kube-v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc.ova", + name: "ova with no version in the product section", + kubernetesTemplateOva: &types.VAppTemplate{ + Name: "dummy", + Children: &types.VAppTemplateChildren{VM: []*types.VAppTemplate{ + { + ProductSection: &types.ProductSection{ + Property: []*types.Property{ + { + Key: "foo", + Value: &types.Value{Value: "bar"}, + }, + }, + }, + }, + }}, + }, + wantErr: "could not find any Version inside the Kubernetes Template OVA 'dummy' Product section properties", + }, + { + name: "correct ova", + kubernetesTemplateOva: &types.VAppTemplate{ + Name: "dummy", + Children: &types.VAppTemplateChildren{VM: []*types.VAppTemplate{ + { + ProductSection: &types.ProductSection{ + Property: []*types.Property{ + { + Key: "VERSION", + Value: &types.Value{Value: "v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc"}, + }, + }, + }, + }, + }}, + }, want: tkgVersionBundle{ EtcdVersion: "v3.5.6_vmware.9", CoreDnsVersion: "v1.9.3_vmware.8", @@ -54,15 +101,15 @@ func Test_getTkgVersionBundleFromVAppTemplateName(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := getTkgVersionBundleFromVAppTemplateName(tt.kubernetesTemplateOvaName) + got, err := getTkgVersionBundleFromVAppTemplate(tt.kubernetesTemplateOva) if err != nil { if tt.wantErr != err.Error() { - t.Errorf("getTkgVersionBundleFromVAppTemplateName() error = %v, wantErr = %v", err, tt.wantErr) + t.Errorf("getTkgVersionBundleFromVAppTemplate() error = %v, wantErr = %v", err, tt.wantErr) } return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getTkgVersionBundleFromVAppTemplateName() got = %v, want %v", got, tt.want) + t.Errorf("getTkgVersionBundleFromVAppTemplate() got = %v, want %v", got, tt.want) } }) } diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 05a01c2e0..09fcec252 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -3,6 +3,7 @@ package govcd import ( "fmt" semver "github.com/hashicorp/go-version" + "github.com/vmware/go-vcloud-director/v2/types/v56" "sigs.k8s.io/yaml" "strings" ) @@ -40,9 +41,9 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", *input.KubernetesTemplateOvaId, err) } // Check the versions of the selected OVA before upgrading - versions, err := getTkgVersionBundleFromVAppTemplateName(vAppTemplate.VAppTemplate.Name) + versions, err := getTkgVersionBundleFromVAppTemplate(vAppTemplate.VAppTemplate) if err != nil { - return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the TKG versions of OVA '%s': %s", vAppTemplate.VAppTemplate.Name, err) + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the TKG versions of OVA '%s': %s", *input.KubernetesTemplateOvaId, err) } if versions.TkgVersion < cluster.capvcdType.Status.Capvcd.Upgrade.Target.TkgVersion { @@ -51,7 +52,7 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) if versions.KubernetesVersion < cluster.capvcdType.Status.Capvcd.Upgrade.Target.KubernetesVersion { return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the new one '%s' has an older TKG version", vAppTemplate.VAppTemplate.Name) } - err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate.Name) + err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate) if err != nil { return cluster.capvcdType.Spec.CapiYaml, err } @@ -82,8 +83,8 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) // used by all the cluster elements. // The caveat here is that not only VCDMachineTemplate needs to be changed with the new OVA name, but also // other fields that reference the related Kubernetes version, TKG version and other derived information. -func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]interface{}, kubernetesTemplateOvaName string) error { - tkgBundle, err := getTkgVersionBundleFromVAppTemplateName(kubernetesTemplateOvaName) +func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]interface{}, kubernetesTemplateOva *types.VAppTemplate) error { + tkgBundle, err := getTkgVersionBundleFromVAppTemplate(kubernetesTemplateOva) if err != nil { return err } @@ -94,7 +95,7 @@ func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]interface{}, k if !ok { return fmt.Errorf("the VCDMachineTemplate 'spec.template.spec.template' field is missing") } - d["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{})["template"] = kubernetesTemplateOvaName + d["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{})["template"] = kubernetesTemplateOva.Name case "MachineDeployment": ok := traverseMapAndGet[string](d, "spec.template.spec.version") != "" if !ok { diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 5a825b346..3c33a52ab 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -4,6 +4,7 @@ package govcd import ( semver "github.com/hashicorp/go-version" + "github.com/vmware/go-vcloud-director/v2/types/v56" "os" "reflect" "strings" @@ -22,36 +23,53 @@ func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { t.Fatalf("could not unmarshal CAPI YAML test file: %s", err) } - // We explore the YAML documents to get the OVA template name that will be updated - // with the new one. - oldOvaName := "" - for _, document := range yamlDocs { - if document["kind"] != "VCDMachineTemplate" { - continue - } - - oldOvaName = traverseMapAndGet[string](document, "spec.template.spec.template") - if oldOvaName == "" { - t.Fatalf("expected to find spec.template.spec.template in %v but got an error: %s", document, err) - } - break - } - if oldOvaName == "" { - t.Fatalf("the OVA that needs to be changed is empty") + oldOvaVersion := "v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc" // Matches the version in capiYaml.yaml + if strings.Count(string(capiYaml), oldOvaVersion) < 2 { + t.Fatalf("the testing YAML doesn't contain the OVA to change") } - oldTkgBundle, err := getTkgVersionBundleFromVAppTemplateName(oldOvaName) + + oldTkgBundle, err := getTkgVersionBundleFromVAppTemplate(&types.VAppTemplate{ + Name: "dummy", + Children: &types.VAppTemplateChildren{VM: []*types.VAppTemplate{ + { + ProductSection: &types.ProductSection{ + Property: []*types.Property{ + { + Key: "VERSION", + Value: &types.Value{Value: oldOvaVersion}, + }, + }, + }, + }, + }}, + }) if err != nil { t.Fatalf("%s", err) } // We call the function to update the old OVA with the new one - newOvaName := "ubuntu-2004-kube-v1.19.16+vmware.1-tkg.2-fba68db15591c15fcd5f26b512663a42" - newTkgBundle, err := getTkgVersionBundleFromVAppTemplateName(newOvaName) + newOva := &types.VAppTemplate{ + ID: "urn:vcloud:vapptemplate:e23b8a5c-e676-4d82-9050-c906a3ac2fea", + Name: "dummy", + Children: &types.VAppTemplateChildren{VM: []*types.VAppTemplate{ + { + ProductSection: &types.ProductSection{ + Property: []*types.Property{ + { + Key: "VERSION", + Value: &types.Value{Value: "v1.19.16+vmware.1-tkg.2-fba68db15591c15fcd5f26b512663a42"}, + }, + }, + }, + }, + }}, + } + newTkgBundle, err := getTkgVersionBundleFromVAppTemplate(newOva) if err != nil { t.Fatalf("%s", err) } - err = cseUpdateKubernetesTemplateInYaml(yamlDocs, newOvaName) + err = cseUpdateKubernetesTemplateInYaml(yamlDocs, newOva) if err != nil { t.Fatalf("%s", err) } @@ -62,7 +80,7 @@ func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { } // No document should have the old OVA - if !strings.Contains(updatedYaml, newOvaName) || strings.Contains(updatedYaml, oldOvaName) { + if strings.Count(updatedYaml, newOva.Name) < 2 || strings.Contains(updatedYaml, oldOvaVersion) { t.Fatalf("failed updating the Kubernetes OVA template:\n%s", updatedYaml) } if !strings.Contains(updatedYaml, newTkgBundle.KubernetesVersion) || strings.Contains(updatedYaml, oldTkgBundle.KubernetesVersion) { diff --git a/types/v56/types.go b/types/v56/types.go index 8bd6292ef..48c66dd7b 100644 --- a/types/v56/types.go +++ b/types/v56/types.go @@ -1582,6 +1582,7 @@ type VAppTemplate struct { NetworkConnectionSection *NetworkConnectionSection `xml:"NetworkConnectionSection,omitempty"` LeaseSettingsSection *LeaseSettingsSection `xml:"LeaseSettingsSection,omitempty"` CustomizationSection *CustomizationSection `xml:"CustomizationSection,omitempty"` + ProductSection *ProductSection `xml:"ProductSection,omitempty"` // OVF Section needs to be added // Section Section `xml:"Section,omitempty"` } From 4c70cad1ff64d9996707f29c577253d85b027b9f Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 9 Feb 2024 13:27:51 +0100 Subject: [PATCH 031/115] Fix getTkgVersionBundleFromVAppTemplate Signed-off-by: abarreiro --- govcd/cse_util.go | 2 +- govcd/cse_util_unit_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 4a5576961..ed4755fd3 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -640,7 +640,7 @@ func getTkgVersionBundleFromVAppTemplate(template *types.VAppTemplate) (tkgVersi } } if id == "" { - return result, fmt.Errorf("could not find any Version inside the Kubernetes Template OVA '%s' Product section properties", template.Name) + return result, fmt.Errorf("could not find any VERSION property inside the Kubernetes Template OVA '%s' Product section", template.Name) } tkgVersionsMap := "cse/tkg_versions.json" diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index 4cc21ca66..720f5c735 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -71,7 +71,7 @@ func Test_getTkgVersionBundleFromVAppTemplate(t *testing.T) { }, }}, }, - wantErr: "could not find any Version inside the Kubernetes Template OVA 'dummy' Product section properties", + wantErr: "could not find any VERSION property inside the Kubernetes Template OVA 'dummy' Product section", }, { name: "correct ova", From cfec8da4e21f78c4f1b12799e63d8f2b19c4eadf Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 9 Feb 2024 14:01:40 +0100 Subject: [PATCH 032/115] Fixes Signed-off-by: abarreiro --- govcd/cse_test.go | 2 +- govcd/cse_util.go | 21 +++++++++++++-------- govcd/cse_util_unit_test.go | 4 ++-- govcd/cse_yaml_unit_test.go | 8 ++++---- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 6173f1611..360ac5614 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -217,7 +217,7 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { org, err := vcd.client.GetOrgByName(vcd.config.Cse.TenantOrg) check.Assert(err, IsNil) - cluster, err := org.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:e8e82bcc-50a1-484f-9dd0-20965ab3e865") + cluster, err := org.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:60e287b2-db49-4316-84c0-e0d3d58e8f52") check.Assert(err, IsNil) workerPoolName := "cse-test1-worker-node-pool-1" diff --git a/govcd/cse_util.go b/govcd/cse_util.go index ed4755fd3..5c7f47327 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -258,15 +258,20 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu result.ControlPlane.DiskSizeGi = diskSizeGi // We retrieve the Kubernetes Template OVA just once for the Control Plane because all YAML blocks share the same - catalog, err := rde.client.GetCatalogByName(result.capvcdType.Status.Capvcd.VcdProperties.Organizations[0].Name, traverseMapAndGet[string](yamlDocument, "spec.template.spec.catalog")) + vAppTemplateName := traverseMapAndGet[string](yamlDocument, "spec.template.spec.template") + catalogName := traverseMapAndGet[string](yamlDocument, "spec.template.spec.catalog") + vAppTemplates, err := queryVappTemplateListWithFilter(rde.client, map[string]string{ + "catalogName": catalogName, + "name": vAppTemplateName, + }) if err != nil { - return nil, err + return nil, fmt.Errorf("could not find any vApp Template with name '%s' in Catalog '%s': %s", vAppTemplateName, catalogName, err) } - ova, err := catalog.GetVAppTemplateByName(traverseMapAndGet[string](yamlDocument, "spec.template.spec.template")) - if err != nil { - return nil, err + if len(vAppTemplates) == 0 { + return nil, fmt.Errorf("could not find any vApp Template with name '%s' in Catalog '%s'", vAppTemplateName, catalogName) } - result.KubernetesTemplateOvaId = ova.VAppTemplate.ID + // The records don't have ID set, so we calculate it + result.KubernetesTemplateOvaId = fmt.Sprintf("urn:vcloud:vapptemplate:%s", extractUuid(vAppTemplates[0].HREF)) } else { // This is one Worker Pool. We need to check the map of worker pools, just in case we already saved the // machine count from MachineDeployment. @@ -635,8 +640,8 @@ func getTkgVersionBundleFromVAppTemplate(template *types.VAppTemplate) (tkgVersi } id := "" for _, prop := range template.Children.VM[0].ProductSection.Property { - if prop != nil && prop.Key == "VERSION" && prop.Value != nil { - id = prop.Value.Value + if prop != nil && prop.Key == "VERSION" { + id = prop.DefaultValue // Use DefaultValue and not Value as the value we want is in the "value" attr } } if id == "" { diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index 720f5c735..5e99683d2 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -63,8 +63,8 @@ func Test_getTkgVersionBundleFromVAppTemplate(t *testing.T) { ProductSection: &types.ProductSection{ Property: []*types.Property{ { - Key: "foo", - Value: &types.Value{Value: "bar"}, + Key: "foo", + DefaultValue: "bar", }, }, }, diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 3c33a52ab..be65bfc99 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -35,8 +35,8 @@ func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { ProductSection: &types.ProductSection{ Property: []*types.Property{ { - Key: "VERSION", - Value: &types.Value{Value: oldOvaVersion}, + Key: "VERSION", + DefaultValue: oldOvaVersion, }, }, }, @@ -56,8 +56,8 @@ func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { ProductSection: &types.ProductSection{ Property: []*types.Property{ { - Key: "VERSION", - Value: &types.Value{Value: "v1.19.16+vmware.1-tkg.2-fba68db15591c15fcd5f26b512663a42"}, + Key: "VERSION", + DefaultValue: "v1.19.16+vmware.1-tkg.2-fba68db15591c15fcd5f26b512663a42", }, }, }, From 40b4f0aa95f5d486e439c0cedd944ddf166a6022 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 9 Feb 2024 16:37:35 +0100 Subject: [PATCH 033/115] Add TKG bundle version comparisons, not tested Signed-off-by: abarreiro --- govcd/catalogitem.go | 5 ++++- govcd/cse.go | 25 ++++++++++++++++++++++--- govcd/cse_test.go | 4 ++++ govcd/cse_util.go | 29 +++++++++++++++++++++++++++++ govcd/cse_yaml.go | 8 ++------ 5 files changed, 61 insertions(+), 10 deletions(-) diff --git a/govcd/catalogitem.go b/govcd/catalogitem.go index 33b48944b..fe321e382 100644 --- a/govcd/catalogitem.go +++ b/govcd/catalogitem.go @@ -108,9 +108,12 @@ func queryVappTemplateListWithFilter(client *Client, filter map[string]string) ( for k, v := range filter { filterEncoded += fmt.Sprintf("%s==%s;", url.QueryEscape(k), url.QueryEscape(v)) } + if len(filterEncoded) > 0 { + filterEncoded = filterEncoded[:len(filterEncoded)-1] // Removes the trailing ';' + } results, err := client.cumulativeQuery(vappTemplateType, nil, map[string]string{ "type": vappTemplateType, - "filter": filterEncoded[:len(filterEncoded)-1], // Removes the trailing ';' + "filter": filterEncoded, }) if err != nil { return nil, fmt.Errorf("error querying vApp templates %s", err) diff --git a/govcd/cse.go b/govcd/cse.go index bdf4fc240..27c8a9afc 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -150,9 +150,28 @@ func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpd }, refresh) } -// GetSupportedUpgrades gets a list of Kubernetes Template OVA IDs that can be used for a cluster upgrade -func (cluster *CseKubernetesCluster) GetSupportedUpgrades() ([]string, error) { - return nil, nil +// GetSupportedUpgrades queries all vApp Templates from VCD and returns those Kubernetes Templates that +// can be used for upgrading the cluster. +func (cluster *CseKubernetesCluster) GetSupportedUpgrades() ([]*types.VAppTemplate, error) { + vAppTemplates, err := queryVappTemplateListWithFilter(cluster.client, nil) + if err != nil { + return nil, fmt.Errorf("could not get vApp Templates: %s", err) + } + var tkgmOvaIds []*types.VAppTemplate + for _, template := range vAppTemplates { + vAppTemplate, err := getVAppTemplateById(cluster.client, fmt.Sprintf("urn:vcloud:vapptemplate:%s", extractUuid(template.HREF))) + if err != nil { + continue // This means we cannot retrieve it (maybe due to some rights missing), so we cannot use it. We skip it + } + tkgVersions, err := getTkgVersionBundleFromVAppTemplate(vAppTemplate.VAppTemplate) + if err != nil { + continue // This means it's not a TKGm OVA, we skip it + } + if tkgVersions.compareTkgVersion(cluster.TkgVersion.String()) == 1 && tkgVersions.kubernetesVersionIsOneMinorHigher(cluster.KubernetesVersion.String()) { + tkgmOvaIds = append(tkgmOvaIds, vAppTemplate.VAppTemplate) + } + } + return tkgmOvaIds, nil } // UpgradeCluster executes an update on the receiver cluster to change the Kubernetes template of the cluster. diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 360ac5614..dbdc767c4 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -220,6 +220,10 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { cluster, err := org.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:60e287b2-db49-4316-84c0-e0d3d58e8f52") check.Assert(err, IsNil) + upgrades, err := cluster.GetSupportedUpgrades() + check.Assert(err, IsNil) + check.Assert(len(upgrades) > 0, Equals, true) + workerPoolName := "cse-test1-worker-node-pool-1" kubeconfig, err := cluster.GetKubeconfig() diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 5c7f47327..22d1d5257 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -674,6 +674,35 @@ func getTkgVersionBundleFromVAppTemplate(template *types.VAppTemplate) (tkgVersi return result, nil } +func (tkgVersions tkgVersionBundle) compareTkgVersion(tkgVersion string) int { + receiverVersion, err := semver.NewVersion(tkgVersions.TkgVersion) + if err != nil { + return -2 + } + inputVersion, err := semver.NewVersion(tkgVersion) + if err != nil { + return -2 + } + return receiverVersion.Compare(inputVersion) +} + +func (tkgVersions tkgVersionBundle) kubernetesVersionIsOneMinorHigher(kubernetesVersion string) bool { + receiverVersionTokens := strings.Split(tkgVersions.KubernetesVersion, ".") + if len(receiverVersionTokens) < 3 { + return false + } + vTokens := strings.Split(kubernetesVersion, ".") + if len(vTokens) < 3 { + return false + } + minor, err := strconv.Atoi(receiverVersionTokens[1]) + if err != nil { + return false + } + + return receiverVersionTokens[0] == vTokens[0] && fmt.Sprintf("%d", minor+1) == vTokens[1] && receiverVersionTokens[2] == vTokens[2] +} + // getVcdKeConfig gets the required information from the CSE Server configuration RDE (VCDKEConfig), such as the // Machine Health Check settings and the Container Registry URL. func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheckActive bool) (*vcdKeConfig, error) { diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 09fcec252..daa698284 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -45,12 +45,8 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) if err != nil { return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the TKG versions of OVA '%s': %s", *input.KubernetesTemplateOvaId, err) } - - if versions.TkgVersion < cluster.capvcdType.Status.Capvcd.Upgrade.Target.TkgVersion { - return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the new one '%s' has an older TKG version", vAppTemplate.VAppTemplate.Name) - } - if versions.KubernetesVersion < cluster.capvcdType.Status.Capvcd.Upgrade.Target.KubernetesVersion { - return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the new one '%s' has an older TKG version", vAppTemplate.VAppTemplate.Name) + if versions.compareTkgVersion(cluster.capvcdType.Status.Capvcd.Upgrade.Target.TkgVersion) != 1 || !versions.kubernetesVersionIsOneMinorHigher(cluster.capvcdType.Status.Capvcd.Upgrade.Target.KubernetesVersion) { + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the new one '%s' has an older TKG/Kubernetes version (%s/%s)", vAppTemplate.VAppTemplate.Name, versions.TkgVersion, versions.KubernetesVersion) } err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate) if err != nil { From ad3e9d595b57c1492e8deae4242d9649d07eb9ce Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 12 Feb 2024 10:51:12 +0100 Subject: [PATCH 034/115] Improvements Signed-off-by: abarreiro --- govcd/cse.go | 35 +++++++++++++++++++++++++---------- govcd/cse_test.go | 2 +- govcd/cse_type.go | 5 +++-- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 27c8a9afc..66304d107 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -150,31 +150,46 @@ func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpd }, refresh) } -// GetSupportedUpgrades queries all vApp Templates from VCD and returns those Kubernetes Templates that -// can be used for upgrading the cluster. -func (cluster *CseKubernetesCluster) GetSupportedUpgrades() ([]*types.VAppTemplate, error) { +// GetSupportedUpgrades queries all vApp Templates from VCD, one by one, and returns those that can be used for upgrading the cluster. +// As retrieving all OVAs one by one from VCD is expensive, the first time this method is called the returned OVAs are +// cached to avoid querying VCD for every OVA whenever this method is called. +// If refresh=true, this cache is cleared out and this method will query VCD for every vApp Template again. +// Therefore, the refresh flag should be set to true only when VCD has new OVAs that need to be considered or the cluster +// has significantly changed since the first call. +func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refresh bool) ([]*types.VAppTemplate, error) { + if refresh { + cluster.supportedUpgrades = nil + } + if len(cluster.supportedUpgrades) != 0 { + return cluster.supportedUpgrades, nil + } + vAppTemplates, err := queryVappTemplateListWithFilter(cluster.client, nil) if err != nil { return nil, fmt.Errorf("could not get vApp Templates: %s", err) } - var tkgmOvaIds []*types.VAppTemplate + var tkgmOvas []*types.VAppTemplate for _, template := range vAppTemplates { + // We can only know if the vApp Template is a TKGm OVA by inspecting its internals, hence we need to retrieve every one + // of them one by one. This is an expensive operation, hence the cache. vAppTemplate, err := getVAppTemplateById(cluster.client, fmt.Sprintf("urn:vcloud:vapptemplate:%s", extractUuid(template.HREF))) if err != nil { continue // This means we cannot retrieve it (maybe due to some rights missing), so we cannot use it. We skip it } - tkgVersions, err := getTkgVersionBundleFromVAppTemplate(vAppTemplate.VAppTemplate) + targetVersions, err := getTkgVersionBundleFromVAppTemplate(vAppTemplate.VAppTemplate) if err != nil { continue // This means it's not a TKGm OVA, we skip it } - if tkgVersions.compareTkgVersion(cluster.TkgVersion.String()) == 1 && tkgVersions.kubernetesVersionIsOneMinorHigher(cluster.KubernetesVersion.String()) { - tkgmOvaIds = append(tkgmOvaIds, vAppTemplate.VAppTemplate) + if targetVersions.compareTkgVersion(cluster.TkgVersion.String()) == 1 && targetVersions.kubernetesVersionIsOneMinorHigher(cluster.KubernetesVersion.String()) { + tkgmOvas = append(tkgmOvas, vAppTemplate.VAppTemplate) } } - return tkgmOvaIds, nil + cluster.supportedUpgrades = tkgmOvas + return tkgmOvas, nil } -// UpgradeCluster executes an update on the receiver cluster to change the Kubernetes template of the cluster. +// UpgradeCluster executes an update on the receiver cluster to upgrade the Kubernetes template of the cluster. +// If the cluster is not upgradeable or the OVA is incorrect, this method will return an error. // If refresh=true, it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) UpgradeCluster(kubernetesTemplateOvaId string, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ @@ -257,7 +272,7 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh break } if err != nil { - // If it's an ETag error, we just retry + // If it's an ETag error, we just retry without waiting if !strings.Contains(strings.ToLower(err.Error()), "etag") { return err } diff --git a/govcd/cse_test.go b/govcd/cse_test.go index dbdc767c4..8f38fed37 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -220,7 +220,7 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { cluster, err := org.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:60e287b2-db49-4316-84c0-e0d3d58e8f52") check.Assert(err, IsNil) - upgrades, err := cluster.GetSupportedUpgrades() + upgrades, err := cluster.GetSupportedUpgrades(true) check.Assert(err, IsNil) check.Assert(len(upgrades) > 0, Equals, true) diff --git a/govcd/cse_type.go b/govcd/cse_type.go index 538070957..961d325a5 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -22,8 +22,9 @@ type CseKubernetesCluster struct { Upgradeable bool Events []CseClusterEvent - client *Client - capvcdType *types.Capvcd + client *Client + capvcdType *types.Capvcd + supportedUpgrades []*types.VAppTemplate // Caches the vApp templates that can be used to upgrade a cluster. } // CseClusterEvent is an event that has occurred during the lifetime of a Container Service Extension (CSE) Kubernetes cluster. From 7c2c7a20eb69518fa3e6e4194976568c88f8ee11 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 12 Feb 2024 10:51:41 +0100 Subject: [PATCH 035/115] Improvements Signed-off-by: abarreiro --- govcd/cse.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 66304d107..7a5a78b8f 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -153,11 +153,11 @@ func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpd // GetSupportedUpgrades queries all vApp Templates from VCD, one by one, and returns those that can be used for upgrading the cluster. // As retrieving all OVAs one by one from VCD is expensive, the first time this method is called the returned OVAs are // cached to avoid querying VCD for every OVA whenever this method is called. -// If refresh=true, this cache is cleared out and this method will query VCD for every vApp Template again. -// Therefore, the refresh flag should be set to true only when VCD has new OVAs that need to be considered or the cluster +// If refreshOvas=true, this cache is cleared out and this method will query VCD for every vApp Template again. +// Therefore, the refreshOvas flag should be set to true only when VCD has new OVAs that need to be considered or the cluster // has significantly changed since the first call. -func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refresh bool) ([]*types.VAppTemplate, error) { - if refresh { +func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refreshOvas bool) ([]*types.VAppTemplate, error) { + if refreshOvas { cluster.supportedUpgrades = nil } if len(cluster.supportedUpgrades) != 0 { From 2b23330865a5d1afbcd446216dd6da8478e89382 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 12 Feb 2024 10:52:39 +0100 Subject: [PATCH 036/115] Improvements Signed-off-by: abarreiro --- govcd/cse.go | 1 + 1 file changed, 1 insertion(+) diff --git a/govcd/cse.go b/govcd/cse.go index 7a5a78b8f..05fb082ba 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -86,6 +86,7 @@ func getCseKubernetesCluster(client *Client, clusterId string) (*CseKubernetesCl } // Refresh gets the latest information about the receiver cluster and updates its properties. +// All cached fields such as the supported OVAs list (from CseKubernetesCluster.GetSupportedUpgrades) are also cleared. func (cluster *CseKubernetesCluster) Refresh() error { refreshed, err := getCseKubernetesCluster(cluster.client, cluster.ID) if err != nil { From 9c74b18f16efd2cc274ebbb4ae5a7274b03a9ec4 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 12 Feb 2024 10:53:47 +0100 Subject: [PATCH 037/115] Improvements Signed-off-by: abarreiro --- govcd/cse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/govcd/cse.go b/govcd/cse.go index 05fb082ba..404809c10 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -153,7 +153,7 @@ func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpd // GetSupportedUpgrades queries all vApp Templates from VCD, one by one, and returns those that can be used for upgrading the cluster. // As retrieving all OVAs one by one from VCD is expensive, the first time this method is called the returned OVAs are -// cached to avoid querying VCD for every OVA whenever this method is called. +// cached to avoid querying VCD again multiple times. // If refreshOvas=true, this cache is cleared out and this method will query VCD for every vApp Template again. // Therefore, the refreshOvas flag should be set to true only when VCD has new OVAs that need to be considered or the cluster // has significantly changed since the first call. From 288cec1fbc9da80b35ef22a6ea7eacdc38b76eb5 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 12 Feb 2024 16:29:26 +0100 Subject: [PATCH 038/115] Add get by name method Signed-off-by: abarreiro --- govcd/cse.go | 23 ++++ govcd/cse/4.2.0/capiyaml_cluster.tmpl | 153 +++++++++++++++++++++++ govcd/cse/4.2.0/capiyaml_mhc.tmpl | 22 ++++ govcd/cse/4.2.0/capiyaml_workerpool.tmpl | 41 ++++++ govcd/cse/4.2.0/rde.tmpl | 31 +++++ govcd/cse_test.go | 5 + govcd/cse_util.go | 22 +++- govcd/cse_util_unit_test.go | 14 ++- 8 files changed, 306 insertions(+), 5 deletions(-) create mode 100644 govcd/cse/4.2.0/capiyaml_cluster.tmpl create mode 100644 govcd/cse/4.2.0/capiyaml_mhc.tmpl create mode 100644 govcd/cse/4.2.0/capiyaml_workerpool.tmpl create mode 100644 govcd/cse/4.2.0/rde.tmpl diff --git a/govcd/cse.go b/govcd/cse.go index 404809c10..b13d64074 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -3,6 +3,7 @@ package govcd import ( "encoding/json" "fmt" + semver "github.com/hashicorp/go-version" "github.com/vmware/go-vcloud-director/v2/types/v56" "github.com/vmware/go-vcloud-director/v2/util" "strings" @@ -76,6 +77,28 @@ func (org *Org) CseGetKubernetesClusterById(id string) (*CseKubernetesCluster, e return getCseKubernetesCluster(org.client, id) } +// CseGetKubernetesClustersByName retrieves the CSE Kubernetes cluster from VCD with the given name +func (org *Org) CseGetKubernetesClustersByName(cseVersion semver.Version, name string) ([]*CseKubernetesCluster, error) { + cseSubcomponents, err := getCseComponentsVersions(cseVersion) + if err != nil { + return nil, err + } + + rdes, err := getRdesByName(org.client, "vmware", "capvcdCluster", cseSubcomponents.CapvcdRdeTypeVersion, name) + if err != nil { + return nil, err + } + clusters := make([]*CseKubernetesCluster, len(rdes)) + for i, rde := range rdes { + cluster, err := cseConvertToCseKubernetesClusterType(rde) + if err != nil { + return nil, err + } + clusters[i] = cluster + } + return clusters, nil +} + // getCseKubernetesCluster retrieves a CSE Kubernetes cluster from VCD by its unique ID func getCseKubernetesCluster(client *Client, clusterId string) (*CseKubernetesCluster, error) { rde, err := getRdeById(client, clusterId) diff --git a/govcd/cse/4.2.0/capiyaml_cluster.tmpl b/govcd/cse/4.2.0/capiyaml_cluster.tmpl new file mode 100644 index 000000000..16a676ae1 --- /dev/null +++ b/govcd/cse/4.2.0/capiyaml_cluster.tmpl @@ -0,0 +1,153 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" + labels: + cluster-role.tkg.tanzu.vmware.com/management: "" + tanzuKubernetesRelease: "{{.TkrVersion}}" + tkg.tanzu.vmware.com/cluster-name: "{{.ClusterName}}" + annotations: + osInfo: "ubuntu,20.04,amd64" + TKGVERSION: "{{.TkgVersion}}" +spec: + clusterNetwork: + pods: + cidrBlocks: + - "{{.PodCidr}}" + serviceDomain: cluster.local + services: + cidrBlocks: + - "{{.ServiceCidr}}" + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "{{.ClusterName}}-control-plane-node-pool" + namespace: "{{.TargetNamespace}}" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: VCDCluster + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" +--- +apiVersion: v1 +kind: Secret +metadata: + name: capi-user-credentials + namespace: {{.TargetNamespace}} +type: Opaque +data: + username: "{{.UsernameB64}}" + refreshToken: "{{.ApiTokenB64}}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: VCDCluster +metadata: + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" +spec: + site: "{{.VcdSite}}" + org: "{{.Org}}" + ovdc: "{{.OrgVdc}}" + ovdcNetwork: "{{.OrgVdcNetwork}}" + {{- if .ControlPlaneEndpoint}} + controlPlaneEndpoint: + host: "{{.ControlPlaneEndpoint}}" + port: 6443 + {{- end}} + {{- if .VirtualIpSubnet}} + loadBalancerConfigSpec: + vipSubnet: "{{.VirtualIpSubnet}}" + {{- end}} + useAsManagementCluster: false + userContext: + secretRef: + name: capi-user-credentials + namespace: "{{.TargetNamespace}}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: VCDMachineTemplate +metadata: + name: "{{.ClusterName}}-control-plane-node-pool" + namespace: "{{.TargetNamespace}}" +spec: + template: + spec: + catalog: "{{.Catalog}}" + template: "{{.VAppTemplate}}" + sizingPolicy: "{{.ControlPlaneSizingPolicy}}" + placementPolicy: "{{.ControlPlanePlacementPolicy}}" + storageProfile: "{{.ControlPlaneStorageProfile}}" + diskSize: {{.ControlPlaneDiskSize}} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "{{.ClusterName}}-control-plane-node-pool" + namespace: "{{.TargetNamespace}}" +spec: + kubeadmConfigSpec: + preKubeadmCommands: + - mv /etc/ssl/certs/custom_certificate_*.crt /usr/local/share/ca-certificates && update-ca-certificates + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + dns: + imageRepository: "{{.ContainerRegistryUrl}}" + imageTag: "{{.DnsVersion}}" + etcd: + local: + imageRepository: "{{.ContainerRegistryUrl}}" + imageTag: "{{.EtcdVersion}}" + imageRepository: "{{.ContainerRegistryUrl}}" + users: + - name: root + sshAuthorizedKeys: + - "{{.SshPublicKey}}" + initConfiguration: + nodeRegistration: + criSocket: /run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cloud-provider: external + joinConfiguration: + nodeRegistration: + criSocket: /run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cloud-provider: external + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: VCDMachineTemplate + name: "{{.ClusterName}}-control-plane-node-pool" + namespace: "{{.TargetNamespace}}" + replicas: {{.ControlPlaneMachineCount}} + version: "{{.KubernetesVersion}}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "{{.ClusterName}}-kct" + namespace: "{{.TargetNamespace}}" +spec: + template: + spec: + users: + - name: root + sshAuthorizedKeys: + - "{{.SshPublicKey}}" + useExperimentalRetryJoin: true + preKubeadmCommands: + - mv /etc/ssl/certs/custom_certificate_*.crt /usr/local/share/ca-certificates && update-ca-certificates + joinConfiguration: + nodeRegistration: + criSocket: /run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cloud-provider: external \ No newline at end of file diff --git a/govcd/cse/4.2.0/capiyaml_mhc.tmpl b/govcd/cse/4.2.0/capiyaml_mhc.tmpl new file mode 100644 index 000000000..d31e4c3ec --- /dev/null +++ b/govcd/cse/4.2.0/capiyaml_mhc.tmpl @@ -0,0 +1,22 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" + labels: + clusterctl.cluster.x-k8s.io: "" + clusterctl.cluster.x-k8s.io/move: "" +spec: + clusterName: "{{.ClusterName}}" + maxUnhealthy: "{{.MaxUnhealthyNodePercentage}}" + nodeStartupTimeout: "{{.NodeStartupTimeout}}" + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: "{{.ClusterName}}" + unhealthyConditions: + - type: Ready + status: Unknown + timeout: "{{.NodeUnknownTimeout}}" + - type: Ready + status: "False" + timeout: "{{.NodeNotReadyTimeout}}" \ No newline at end of file diff --git a/govcd/cse/4.2.0/capiyaml_workerpool.tmpl b/govcd/cse/4.2.0/capiyaml_workerpool.tmpl new file mode 100644 index 000000000..e2292c7d7 --- /dev/null +++ b/govcd/cse/4.2.0/capiyaml_workerpool.tmpl @@ -0,0 +1,41 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: VCDMachineTemplate +metadata: + name: "{{.NodePoolName}}" + namespace: "{{.TargetNamespace}}" +spec: + template: + spec: + catalog: "{{.Catalog}}" + template: "{{.VAppTemplate}}" + sizingPolicy: "{{.NodePoolSizingPolicy}}" + placementPolicy: "{{.NodePoolPlacementPolicy}}" + storageProfile: "{{.NodePoolStorageProfile}}" + diskSize: "{{.NodePoolDiskSize}}" + enableNvidiaGPU: {{.NodePoolEnableGpu}} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "{{.NodePoolName}}" + namespace: "{{.TargetNamespace}}" +spec: + clusterName: "{{.ClusterName}}" + replicas: {{.NodePoolMachineCount}} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: "{{.ClusterName}}-kct" + namespace: "{{.TargetNamespace}}" + clusterName: "{{.ClusterName}}" + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: VCDMachineTemplate + name: "{{.NodePoolName}}" + namespace: "{{.TargetNamespace}}" + version: "{{.KubernetesVersion}}" \ No newline at end of file diff --git a/govcd/cse/4.2.0/rde.tmpl b/govcd/cse/4.2.0/rde.tmpl new file mode 100644 index 000000000..e5ea3e2b8 --- /dev/null +++ b/govcd/cse/4.2.0/rde.tmpl @@ -0,0 +1,31 @@ +{ + "apiVersion": "capvcd.vmware.com/v1.1", + "kind": "CAPVCDCluster", + "name": "{{.Name}}", + "metadata": { + "name": "{{.Name}}", + "orgName": "{{.Org}}", + "site": "{{.VcdUrl}}", + "virtualDataCenterName": "{{.Vdc}}" + }, + "spec": { + "vcdKe": { + "isVCDKECluster": true, + "markForDelete": {{.Delete}}, + "forceDelete": {{.ForceDelete}}, + "autoRepairOnErrors": {{.AutoRepairOnErrors}}, + {{- if .DefaultStorageClassName }} + "defaultStorageClassOptions": { + "filesystem": "{{.DefaultStorageClassFileSystem}}", + "k8sStorageClassName": "{{.DefaultStorageClassName}}", + "vcdStorageProfileName": "{{.DefaultStorageClassStorageProfile}}", + "useDeleteReclaimPolicy": {{.DefaultStorageClassUseDeleteReclaimPolicy}} + }, + {{- end }} + "secure": { + "apiToken": "{{.ApiToken}}" + } + }, + "capiYaml": "{{.CapiYaml}}" + } +} diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 8f38fed37..523344742 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -184,6 +184,11 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(cluster.Upgradeable, Equals, clusterGet.Upgradeable) check.Assert(cluster.State, Equals, clusterGet.State) + allClusters, err := org.CseGetKubernetesClustersByName(clusterGet.CseVersion, clusterGet.Name) + check.Assert(err, IsNil) + check.Assert(len(allClusters), Equals, 1) + check.Assert(allClusters[0], DeepEquals, clusterGet) + // Update worker pool from 1 node to 2 // Pre-check. This should be 1, as it was created with just 1 pool for _, nodePool := range cluster.capvcdType.Status.Capvcd.NodePool { diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 22d1d5257..4210ce011 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -16,14 +16,22 @@ import ( // getCseComponentsVersions gets the versions of the subcomponents that are part of Container Service Extension. func getCseComponentsVersions(cseVersion semver.Version) (*cseComponentsVersions, error) { - v41, _ := semver.NewVersion("4.1") + v41, _ := semver.NewVersion("4.1.0") + v42, _ := semver.NewVersion("4.2.0") + v43, _ := semver.NewVersion("4.3.0") - if cseVersion.Equal(v41) { + if cseVersion.GreaterThanOrEqual(v41) && cseVersion.LessThan(v42) { return &cseComponentsVersions{ VcdKeConfigRdeTypeVersion: "1.1.0", CapvcdRdeTypeVersion: "1.2.0", CseInterfaceVersion: "1.0.0", }, nil + } else if cseVersion.GreaterThanOrEqual(v42) && cseVersion.LessThan(v43) { + return &cseComponentsVersions{ + VcdKeConfigRdeTypeVersion: "1.1.0", + CapvcdRdeTypeVersion: "1.3.0", + CseInterfaceVersion: "1.0.0", + }, nil } return nil, fmt.Errorf("the Container Service Extension version '%s' is not supported", cseVersion.String()) } @@ -752,10 +760,16 @@ func getCseTemplate(cseVersion semver.Version, templateName string) (string, err return "", fmt.Errorf("the Container Service minimum version is '%s'", minimumVersion.String()) } versionSegments := cseVersion.Segments() - fullTemplatePath := fmt.Sprintf("cse/%d.%d/%s.tmpl", versionSegments[0], versionSegments[1], templateName) + // We try with major.minor.patch + fullTemplatePath := fmt.Sprintf("cse/%d.%d.%d/%s.tmpl", versionSegments[0], versionSegments[1], versionSegments[2], templateName) result, err := cseFiles.ReadFile(fullTemplatePath) if err != nil { - return "", fmt.Errorf("could not read Go template '%s'", fullTemplatePath) + // We try now just with major.minor + fullTemplatePath = fmt.Sprintf("cse/%d.%d/%s.tmpl", versionSegments[0], versionSegments[1], templateName) + result, err = cseFiles.ReadFile(fullTemplatePath) + if err != nil { + return "", fmt.Errorf("could not read Go template '%s.tmpl' for CSE version %s", templateName, cseVersion.String()) + } } return string(result), nil } diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index 5e99683d2..d76e41287 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -132,6 +132,10 @@ func Test_getCseTemplate(t *testing.T) { if err != nil { t.Fatalf("could not create 4.1.1 version object") } + v420, err := semver.NewVersion("4.2.0") + if err != nil { + t.Fatalf("could not create 4.2.0 version object") + } tmpl41, err := getCseTemplate(*v41, "rde") if err != nil { @@ -145,10 +149,18 @@ func Test_getCseTemplate(t *testing.T) { if err != nil { t.Fatalf("%s", err) } - if tmpl41 != tmpl410 || tmpl41 != tmpl411 || tmpl410 != tmpl411 { + if tmpl41 == "" || tmpl41 != tmpl410 || tmpl41 != tmpl411 || tmpl410 != tmpl411 { t.Fatalf("templates should be the same:\n4.1: %s\n4.1.0: %s\n4.1.1: %s", tmpl41, tmpl410, tmpl411) } + tmpl420, err := getCseTemplate(*v420, "rde") + if err != nil { + t.Fatalf("%s", err) + } + if tmpl420 == "" { + t.Fatalf("the obtained template for %s is empty", v420.String()) + } + _, err = getCseTemplate(*v40, "rde") if err == nil && err.Error() != "the Container Service minimum version is '4.1.0'" { t.Fatalf("expected an error but got %s", err) From f9f541855304037848eb15f7267bcef100abc27a Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 12 Feb 2024 17:27:33 +0100 Subject: [PATCH 039/115] Refactor Signed-off-by: abarreiro --- govcd/cse.go | 26 ++++++++++++++------------ govcd/cse_test.go | 7 ++----- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index b13d64074..e10d29db1 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -30,7 +30,7 @@ func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterSettings, timeo return &CseKubernetesCluster{ID: clusterId}, err } - return org.CseGetKubernetesClusterById(clusterId) + return getCseKubernetesClusterById(org.client, clusterId) } // CseCreateKubernetesClusterAsync creates a Kubernetes cluster with the data given as input (CseClusterSettings), but does not @@ -73,8 +73,8 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) } // CseGetKubernetesClusterById retrieves a CSE Kubernetes cluster from VCD by its unique ID -func (org *Org) CseGetKubernetesClusterById(id string) (*CseKubernetesCluster, error) { - return getCseKubernetesCluster(org.client, id) +func (vcdClient *VCDClient) CseGetKubernetesClusterById(id string) (*CseKubernetesCluster, error) { + return getCseKubernetesClusterById(&vcdClient.Client, id) } // CseGetKubernetesClustersByName retrieves the CSE Kubernetes cluster from VCD with the given name @@ -88,19 +88,21 @@ func (org *Org) CseGetKubernetesClustersByName(cseVersion semver.Version, name s if err != nil { return nil, err } - clusters := make([]*CseKubernetesCluster, len(rdes)) - for i, rde := range rdes { - cluster, err := cseConvertToCseKubernetesClusterType(rde) - if err != nil { - return nil, err + var clusters []*CseKubernetesCluster + for _, rde := range rdes { + if rde.DefinedEntity.Org != nil && rde.DefinedEntity.Org.ID == org.Org.ID { + cluster, err := cseConvertToCseKubernetesClusterType(rde) + if err != nil { + return nil, err + } + clusters = append(clusters, cluster) } - clusters[i] = cluster } return clusters, nil } -// getCseKubernetesCluster retrieves a CSE Kubernetes cluster from VCD by its unique ID -func getCseKubernetesCluster(client *Client, clusterId string) (*CseKubernetesCluster, error) { +// getCseKubernetesClusterById retrieves a CSE Kubernetes cluster from VCD by its unique ID +func getCseKubernetesClusterById(client *Client, clusterId string) (*CseKubernetesCluster, error) { rde, err := getRdeById(client, clusterId) if err != nil { return nil, err @@ -111,7 +113,7 @@ func getCseKubernetesCluster(client *Client, clusterId string) (*CseKubernetesCl // Refresh gets the latest information about the receiver cluster and updates its properties. // All cached fields such as the supported OVAs list (from CseKubernetesCluster.GetSupportedUpgrades) are also cleared. func (cluster *CseKubernetesCluster) Refresh() error { - refreshed, err := getCseKubernetesCluster(cluster.client, cluster.ID) + refreshed, err := getCseKubernetesClusterById(cluster.client, cluster.ID) if err != nil { return fmt.Errorf("failed refreshing the CSE Kubernetes Cluster: %s", err) } diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 523344742..a36ef6d00 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -151,7 +151,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { err = cluster.Refresh() check.Assert(err, IsNil) - clusterGet, err := org.CseGetKubernetesClusterById(cluster.ID) + clusterGet, err := vcd.client.CseGetKubernetesClusterById(cluster.ID) check.Assert(err, IsNil) check.Assert(cluster.CseVersion.String(), Equals, clusterGet.CseVersion.String()) check.Assert(cluster.Name, Equals, clusterGet.Name) @@ -219,10 +219,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { } func (vcd *TestVCD) Test_Deleteme(check *C) { - org, err := vcd.client.GetOrgByName(vcd.config.Cse.TenantOrg) - check.Assert(err, IsNil) - - cluster, err := org.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:60e287b2-db49-4316-84c0-e0d3d58e8f52") + cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:60e287b2-db49-4316-84c0-e0d3d58e8f52") check.Assert(err, IsNil) upgrades, err := cluster.GetSupportedUpgrades(true) From b9066b4523313ae87bd1b228f5f2c492f7b5a9aa Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 13 Feb 2024 16:22:48 +0100 Subject: [PATCH 040/115] Fix delete Signed-off-by: abarreiro --- govcd/cse.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index e10d29db1..fe244f899 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -329,17 +329,9 @@ func (cluster *CseKubernetesCluster) Delete(timeoutMinutes time.Duration) error return fmt.Errorf("could not retrieve the Kubernetes cluster with ID '%s': %s", cluster.ID, err) } - spec, ok := rde.DefinedEntity.Entity["spec"].(map[string]interface{}) - if !ok { - return fmt.Errorf("JSON object 'spec' is not correct in the RDE '%s': %s", cluster.ID, err) - } - - vcdKe, ok = spec["vcdKe"].(map[string]interface{}) - if !ok { - return fmt.Errorf("JSON object 'spec.vcdKe' is not correct in the RDE '%s': %s", cluster.ID, err) - } - - if !vcdKe["markForDelete"].(bool) || !vcdKe["forceDelete"].(bool) { + markForDelete := traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.markForDelete") + forceDelete := traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.forceDelete") + if !markForDelete || !forceDelete { // Mark the cluster for deletion vcdKe["markForDelete"] = true vcdKe["forceDelete"] = true From 385e518769f8ab9a55404aab63560b6b2358d717 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 13 Feb 2024 16:29:58 +0100 Subject: [PATCH 041/115] Make auto-repair-on-errors non updatable Signed-off-by: abarreiro --- govcd/cse.go | 12 ------------ govcd/cse_type.go | 1 - 2 files changed, 13 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index fe244f899..3258f9689 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -231,14 +231,6 @@ func (cluster *CseKubernetesCluster) SetHealthCheck(healthCheckEnabled bool, ref }, refresh) } -// SetAutoRepairOnErrors executes an update on the receiver cluster to change the flag that controls the auto-repair -// capabilities of CSE. If refresh=true, it retrieves the latest state of the cluster from VCD before updating. -func (cluster *CseKubernetesCluster) SetAutoRepairOnErrors(autoRepairOnErrors bool, refresh bool) error { - return cluster.Update(CseClusterUpdateInput{ - AutoRepairOnErrors: &autoRepairOnErrors, - }, refresh) -} - // Update executes an update on the receiver CSE Kubernetes Cluster on any of the allowed parameters defined in the input type. If refresh=true, // it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh bool) error { @@ -256,10 +248,6 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh return fmt.Errorf("can't update a Kubernetes cluster that is not in 'provisioned' state, as it is in '%s'", cluster.capvcdType.Status.VcdKe.State) } - if input.AutoRepairOnErrors != nil { - cluster.capvcdType.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors - } - // Computed attributes that are required, such as the VcdKeConfig version input.clusterName = cluster.Name input.vcdKeConfigVersion = cluster.capvcdType.Status.VcdKe.VcdKeVersion diff --git a/govcd/cse_type.go b/govcd/cse_type.go index 961d325a5..c69e39de2 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -92,7 +92,6 @@ type CseClusterUpdateInput struct { WorkerPools *map[string]CseWorkerPoolUpdateInput // Maps a node pool name with its contents NewWorkerPools *[]CseWorkerPoolSettings NodeHealthCheck *bool - AutoRepairOnErrors *bool // Private fields that are computed, not requested to the consumer of this struct vcdKeConfigVersion string From 94ab65d0baae09a728e663f197be09a43e62e0c1 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 13 Feb 2024 16:38:25 +0100 Subject: [PATCH 042/115] Make auto-repair-on-errors non updatable since 4.1.1 Signed-off-by: abarreiro --- govcd/cse.go | 22 ++++++++++++++++++++++ govcd/cse_type.go | 1 + 2 files changed, 23 insertions(+) diff --git a/govcd/cse.go b/govcd/cse.go index 3258f9689..4c3cc80a8 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -231,6 +231,14 @@ func (cluster *CseKubernetesCluster) SetHealthCheck(healthCheckEnabled bool, ref }, refresh) } +// SetAutoRepairOnErrors executes an update on the receiver cluster to change the flag that controls the auto-repair +// capabilities of CSE. If refresh=true, it retrieves the latest state of the cluster from VCD before updating. +func (cluster *CseKubernetesCluster) SetAutoRepairOnErrors(autoRepairOnErrors bool, refresh bool) error { + return cluster.Update(CseClusterUpdateInput{ + AutoRepairOnErrors: &autoRepairOnErrors, + }, refresh) +} + // Update executes an update on the receiver CSE Kubernetes Cluster on any of the allowed parameters defined in the input type. If refresh=true, // it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh bool) error { @@ -248,6 +256,19 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh return fmt.Errorf("can't update a Kubernetes cluster that is not in 'provisioned' state, as it is in '%s'", cluster.capvcdType.Status.VcdKe.State) } + if input.AutoRepairOnErrors != nil { + // Since CSE 4.1.1, the AutoRepairOnError toggle can't be modified and is turned off + // automatically by the CSE Server. + v411, err := semver.NewVersion("4.1.1") + if err != nil { + return fmt.Errorf("can't update the Kubernetes cluster: %s", err) + } + if cluster.CseVersion.GreaterThanOrEqual(v411) { + return fmt.Errorf("the 'Auto Repair on Errors' feature can't be changed after the cluster is created since CSE 4.1.1") + } + cluster.capvcdType.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors + } + // Computed attributes that are required, such as the VcdKeConfig version input.clusterName = cluster.Name input.vcdKeConfigVersion = cluster.capvcdType.Status.VcdKe.VcdKeVersion @@ -319,6 +340,7 @@ func (cluster *CseKubernetesCluster) Delete(timeoutMinutes time.Duration) error markForDelete := traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.markForDelete") forceDelete := traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.forceDelete") + if !markForDelete || !forceDelete { // Mark the cluster for deletion vcdKe["markForDelete"] = true diff --git a/govcd/cse_type.go b/govcd/cse_type.go index c69e39de2..961d325a5 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -92,6 +92,7 @@ type CseClusterUpdateInput struct { WorkerPools *map[string]CseWorkerPoolUpdateInput // Maps a node pool name with its contents NewWorkerPools *[]CseWorkerPoolSettings NodeHealthCheck *bool + AutoRepairOnErrors *bool // Private fields that are computed, not requested to the consumer of this struct vcdKeConfigVersion string From 1c84371ae8c793a53d6a4caee4eccb2ab5f05ae0 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 13 Feb 2024 18:10:36 +0100 Subject: [PATCH 043/115] Add events Signed-off-by: abarreiro --- govcd/cse/4.2.0/rde.tmpl | 6 +++- govcd/cse_util.go | 72 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/govcd/cse/4.2.0/rde.tmpl b/govcd/cse/4.2.0/rde.tmpl index e5ea3e2b8..e44d9394c 100644 --- a/govcd/cse/4.2.0/rde.tmpl +++ b/govcd/cse/4.2.0/rde.tmpl @@ -26,6 +26,10 @@ "apiToken": "{{.ApiToken}}" } }, - "capiYaml": "{{.CapiYaml}}" + "capiYaml": "{{.CapiYaml}}", + "projector": { "operations": + [ + ] + }, } } diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 4210ce011..a407a8836 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -9,6 +9,7 @@ import ( "github.com/vmware/go-vcloud-director/v2/util" "net/url" "regexp" + "sort" "strconv" "strings" "time" @@ -73,6 +74,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu ClusterResourceSetBindings: make([]string, len(capvcd.Status.Capvcd.ClusterResourceSetBindings)), Upgradeable: capvcd.Status.Capvcd.Upgrade.Target.KubernetesVersion != "" && capvcd.Status.Capvcd.Upgrade.Target.TkgVersion != "", State: capvcd.Status.VcdKe.State, + Events: make([]CseClusterEvent, 0), client: rde.client, capvcdType: capvcd, } @@ -341,6 +343,76 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu i++ } + // Add all events to the resulting cluster + for _, s := range capvcd.Status.VcdKe.EventSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "event", + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedEvent, + }) + } + for _, s := range capvcd.Status.VcdKe.ErrorSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "error", + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedError, + }) + } + for _, s := range capvcd.Status.Capvcd.EventSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "event", + OccurredAt: s.OccurredAt, + Details: s.Name, + }) + } + for _, s := range capvcd.Status.Capvcd.ErrorSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "error", + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedError, + }) + } + for _, s := range capvcd.Status.Cpi.EventSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "event", + OccurredAt: s.OccurredAt, + Details: s.Name, + }) + } + for _, s := range capvcd.Status.Cpi.ErrorSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "error", + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedError, + }) + } + for _, s := range capvcd.Status.Csi.EventSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "event", + OccurredAt: s.OccurredAt, + Details: s.Name, + }) + } + for _, s := range capvcd.Status.Csi.ErrorSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "error", + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedError, + }) + } + + sort.SliceStable(result.Events, func(i, j int) bool { + return result.Events[i].OccurredAt.Before(result.Events[j].OccurredAt) + }) + return result, nil } From 60d7a78c74eacaca2df2b31cd8a9742e55689fe6 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 14 Feb 2024 13:17:58 +0100 Subject: [PATCH 044/115] Fixes Signed-off-by: abarreiro --- govcd/cse.go | 4 +- govcd/cse/4.2.0/rde.tmpl | 5 +- govcd/cse_test.go | 44 +++---- govcd/cse_type.go | 11 +- govcd/cse_util.go | 272 ++++++++++++++++++++++----------------- govcd/cse_yaml.go | 5 +- types/v56/cse.go | 8 +- 7 files changed, 187 insertions(+), 162 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 4c3cc80a8..f0cb151ec 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -204,7 +204,7 @@ func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refreshOvas bool) ([]* } targetVersions, err := getTkgVersionBundleFromVAppTemplate(vAppTemplate.VAppTemplate) if err != nil { - continue // This means it's not a TKGm OVA, we skip it + continue // This means it's not a TKGm OVA, or it is not supported, so we skip it } if targetVersions.compareTkgVersion(cluster.TkgVersion.String()) == 1 && targetVersions.kubernetesVersionIsOneMinorHigher(cluster.KubernetesVersion.String()) { tkgmOvas = append(tkgmOvas, vAppTemplate.VAppTemplate) @@ -289,7 +289,7 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh return err } - // We do this loop to increase the chances that the Kubernetes cluster is successfully created, as the Go SDK is + // We do this loop to increase the chances that the Kubernetes cluster is successfully updated, as the Go SDK is // "fighting" with the CSE Server retries := 0 maxRetries := 5 diff --git a/govcd/cse/4.2.0/rde.tmpl b/govcd/cse/4.2.0/rde.tmpl index e44d9394c..7eb0011b3 100644 --- a/govcd/cse/4.2.0/rde.tmpl +++ b/govcd/cse/4.2.0/rde.tmpl @@ -27,9 +27,6 @@ } }, "capiYaml": "{{.CapiYaml}}", - "projector": { "operations": - [ - ] - }, + "projector": { "operations": [] } } } diff --git a/govcd/cse_test.go b/govcd/cse_test.go index a36ef6d00..085bf8ec1 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -136,7 +136,6 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(cluster.CapvcdVersion, Not(Equals), "") check.Assert(cluster.CpiVersion, Not(Equals), "") check.Assert(cluster.CsiVersion, Not(Equals), "") - check.Assert(cluster.Upgradeable, Equals, true) check.Assert(len(cluster.ClusterResourceSetBindings), Not(Equals), 0) check.Assert(cluster.State, Equals, "provisioned") check.Assert(len(cluster.Events), Not(Equals), 0) @@ -181,7 +180,6 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(cluster.ClusterResourceSetBindings, DeepEquals, clusterGet.ClusterResourceSetBindings) check.Assert(cluster.CpiVersion.String(), Equals, clusterGet.CpiVersion.String()) check.Assert(cluster.CsiVersion.String(), Equals, clusterGet.CsiVersion.String()) - check.Assert(cluster.Upgradeable, Equals, clusterGet.Upgradeable) check.Assert(cluster.State, Equals, clusterGet.State) allClusters, err := org.CseGetKubernetesClustersByName(clusterGet.CseVersion, clusterGet.Name) @@ -219,25 +217,25 @@ func (vcd *TestVCD) Test_Cse(check *C) { } func (vcd *TestVCD) Test_Deleteme(check *C) { - cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:60e287b2-db49-4316-84c0-e0d3d58e8f52") + cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:2d137956-e702-474e-a4df-7a51c868f22c") check.Assert(err, IsNil) - upgrades, err := cluster.GetSupportedUpgrades(true) + _, err = cluster.GetSupportedUpgrades(true) check.Assert(err, IsNil) - check.Assert(len(upgrades) > 0, Equals, true) + // check.Assert(len(upgrades) > 0, Equals, true) - workerPoolName := "cse-test1-worker-node-pool-1" + workerPoolName := "node-pool-1" - kubeconfig, err := cluster.GetKubeconfig() - check.Assert(err, IsNil) - check.Assert(true, Equals, strings.Contains(kubeconfig, cluster.Name)) - check.Assert(true, Equals, strings.Contains(kubeconfig, "client-certificate-data")) - check.Assert(true, Equals, strings.Contains(kubeconfig, "certificate-authority-data")) - check.Assert(true, Equals, strings.Contains(kubeconfig, "client-key-data")) + //kubeconfig, err := cluster.GetKubeconfig() + //check.Assert(err, IsNil) + //check.Assert(true, Equals, strings.Contains(kubeconfig, cluster.Name)) + //check.Assert(true, Equals, strings.Contains(kubeconfig, "client-certificate-data")) + //check.Assert(true, Equals, strings.Contains(kubeconfig, "certificate-authority-data")) + //check.Assert(true, Equals, strings.Contains(kubeconfig, "client-key-data")) // Perform the update - err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, true) - check.Assert(err, IsNil) + // err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, true) + // check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up foundWorkerPool := false @@ -250,20 +248,20 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { check.Assert(foundWorkerPool, Equals, true) // Revert back (resources can be limited) - err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 1}}, true) - check.Assert(err, IsNil) + // err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 1}}, true) + // check.Assert(err, IsNil) // Perform the update - err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 2}, true) - check.Assert(err, IsNil) + // err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 2}, true) + // check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up - check.Assert(cluster.ControlPlane.MachineCount, Equals, 2) + // check.Assert(cluster.ControlPlane.MachineCount, Equals, 2) // Revert back (resources can be limited) - err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) - check.Assert(err, IsNil) + // err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) + // check.Assert(err, IsNil) - err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) - check.Assert(err, IsNil) + // err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) + // check.Assert(err, IsNil) } diff --git a/govcd/cse_type.go b/govcd/cse_type.go index 961d325a5..1be10ca60 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -19,7 +19,6 @@ type CseKubernetesCluster struct { CpiVersion semver.Version CsiVersion semver.Version State string - Upgradeable bool Events []CseClusterEvent client *Client @@ -29,10 +28,12 @@ type CseKubernetesCluster struct { // CseClusterEvent is an event that has occurred during the lifetime of a Container Service Extension (CSE) Kubernetes cluster. type CseClusterEvent struct { - Name string - Type string - OccurredAt time.Time - Details string + Name string + Type string + ResourceId string + ResourceName string + OccurredAt time.Time + Details string } // CseClusterSettings defines the required configuration of a Container Service Extension (CSE) Kubernetes cluster. diff --git a/govcd/cse_util.go b/govcd/cse_util.go index a407a8836..4e78b5c1c 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -72,63 +72,173 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu ID: rde.DefinedEntity.ID, Etag: rde.Etag, ClusterResourceSetBindings: make([]string, len(capvcd.Status.Capvcd.ClusterResourceSetBindings)), - Upgradeable: capvcd.Status.Capvcd.Upgrade.Target.KubernetesVersion != "" && capvcd.Status.Capvcd.Upgrade.Target.TkgVersion != "", State: capvcd.Status.VcdKe.State, Events: make([]CseClusterEvent, 0), client: rde.client, capvcdType: capvcd, } - version, err := semver.NewVersion(capvcd.Status.Capvcd.Upgrade.Current.KubernetesVersion) - if err != nil { - return nil, fmt.Errorf("could not read Kubernetes version: %s", err) + + // Add all events to the resulting cluster + for _, s := range capvcd.Status.VcdKe.EventSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "event", + ResourceId: s.VcdResourceId, + ResourceName: s.VcdResourceName, + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedEvent, + }) } - result.KubernetesVersion = *version + for _, s := range capvcd.Status.VcdKe.ErrorSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "error", + ResourceId: s.VcdResourceId, + ResourceName: s.VcdResourceName, + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedError, + }) + } + for _, s := range capvcd.Status.Capvcd.EventSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "event", + ResourceId: s.VcdResourceId, + ResourceName: s.VcdResourceName, + OccurredAt: s.OccurredAt, + Details: s.Name, + }) + } + for _, s := range capvcd.Status.Capvcd.ErrorSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "error", + ResourceId: s.VcdResourceId, + ResourceName: s.VcdResourceName, + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedError, + }) + } + for _, s := range capvcd.Status.Cpi.EventSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "event", + ResourceId: s.VcdResourceId, + ResourceName: s.VcdResourceName, + OccurredAt: s.OccurredAt, + Details: s.Name, + }) + } + for _, s := range capvcd.Status.Cpi.ErrorSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "error", + ResourceId: s.VcdResourceId, + ResourceName: s.VcdResourceName, + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedError, + }) + } + for _, s := range capvcd.Status.Csi.EventSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "event", + ResourceId: s.VcdResourceId, + ResourceName: s.VcdResourceName, + OccurredAt: s.OccurredAt, + Details: s.Name, + }) + } + for _, s := range capvcd.Status.Csi.ErrorSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "error", + ResourceId: s.VcdResourceId, + ResourceName: s.VcdResourceName, + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedError, + }) + } + sort.SliceStable(result.Events, func(i, j int) bool { + return result.Events[i].OccurredAt.After(result.Events[j].OccurredAt) + }) - version, err = semver.NewVersion(capvcd.Status.Capvcd.Upgrade.Current.TkgVersion) - if err != nil { - return nil, fmt.Errorf("could not read Tkg version: %s", err) + if capvcd.Status.Capvcd.Upgrade.Current.KubernetesVersion != "" { + version, err := semver.NewVersion(capvcd.Status.Capvcd.Upgrade.Current.KubernetesVersion) + if err != nil { + return nil, fmt.Errorf("could not read Kubernetes version: %s", err) + } + result.KubernetesVersion = *version } - result.TkgVersion = *version - version, err = semver.NewVersion(capvcd.Status.Capvcd.CapvcdVersion) - if err != nil { - return nil, fmt.Errorf("could not read Capvcd version: %s", err) + if capvcd.Status.Capvcd.Upgrade.Current.TkgVersion != "" { + version, err := semver.NewVersion(capvcd.Status.Capvcd.Upgrade.Current.TkgVersion) + if err != nil { + return nil, fmt.Errorf("could not read Tkg version: %s", err) + } + result.TkgVersion = *version } - result.CapvcdVersion = *version - version, err = semver.NewVersion(strings.TrimSpace(capvcd.Status.Cpi.Version)) // Note: We use trim as the version comes with spacing characters - if err != nil { - return nil, fmt.Errorf("could not read CPI version: %s", err) + if capvcd.Status.Capvcd.CapvcdVersion != "" { + version, err := semver.NewVersion(capvcd.Status.Capvcd.CapvcdVersion) + if err != nil { + return nil, fmt.Errorf("could not read Capvcd version: %s", err) + } + result.CapvcdVersion = *version } - result.CpiVersion = *version - version, err = semver.NewVersion(capvcd.Status.Csi.Version) - if err != nil { - return nil, fmt.Errorf("could not read CSI version: %s", err) + if capvcd.Status.Cpi.Version != "" { + version, err := semver.NewVersion(strings.TrimSpace(capvcd.Status.Cpi.Version)) // Note: We use trim as the version comes with spacing characters + if err != nil { + return nil, fmt.Errorf("could not read CPI version: %s", err) + } + result.CpiVersion = *version + } + + if capvcd.Status.Csi.Version != "" { + version, err := semver.NewVersion(capvcd.Status.Csi.Version) + if err != nil { + return nil, fmt.Errorf("could not read CSI version: %s", err) + } + result.CsiVersion = *version + } + + if capvcd.Status.VcdKe.VcdKeVersion != "" { + cseVersion, err := semver.NewVersion(capvcd.Status.VcdKe.VcdKeVersion) + if err != nil { + return nil, fmt.Errorf("could not read the CSE Version that the cluster uses: %s", err) + } + result.CseVersion = *cseVersion + } + + // Retrieve the Owner + if rde.DefinedEntity.Owner != nil { + result.Owner = rde.DefinedEntity.Owner.Name } - result.CsiVersion = *version // Retrieve the Organization ID for i, binding := range capvcd.Status.Capvcd.ClusterResourceSetBindings { result.ClusterResourceSetBindings[i] = binding.Name } - if len(capvcd.Status.Capvcd.ClusterApiStatus.ApiEndpoints) == 0 { - return nil, fmt.Errorf("could not get Control Plane endpoint") + if len(capvcd.Status.Capvcd.ClusterApiStatus.ApiEndpoints) > 0 { + result.ControlPlane.Ip = capvcd.Status.Capvcd.ClusterApiStatus.ApiEndpoints[0].Host } - result.ControlPlane.Ip = capvcd.Status.Capvcd.ClusterApiStatus.ApiEndpoints[0].Host - if len(result.capvcdType.Status.Capvcd.VcdProperties.Organizations) == 0 { - return nil, fmt.Errorf("could not read Organizations from Capvcd type") + if len(result.capvcdType.Status.Capvcd.VcdProperties.Organizations) > 0 { + result.OrganizationId = result.capvcdType.Status.Capvcd.VcdProperties.Organizations[0].Id } - result.OrganizationId = result.capvcdType.Status.Capvcd.VcdProperties.Organizations[0].Id - // Retrieve the VDC ID + // If the Org/VDC information is not set, we can't continue retrieving information for the cluster. + // This scenario is when the cluster is not correctly provisioned (Error state) if len(result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs) == 0 { - return nil, fmt.Errorf("could not read VDCs from Capvcd type") + return result, nil } - result.VdcId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Id + // NOTE: The code below, until the end of this function, requires the Org/VDC information + + // Retrieve the VDC ID + result.VdcId = result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Id // FIXME: This is a workaround, because for some reason the OrgVdcs[*].Id property contains the VDC name instead of the VDC ID. // Once this is fixed, this conditional should not be needed anymore. if result.VdcId == result.capvcdType.Status.Capvcd.VcdProperties.OrgVdcs[0].Name { @@ -162,19 +272,6 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu } result.NetworkId = networks[0].OpenApiOrgVdcNetwork.ID - // Get the CSE version - cseVersion, err := semver.NewVersion(capvcd.Status.VcdKe.VcdKeVersion) - if err != nil { - return nil, fmt.Errorf("could not read the CSE Version that the cluster uses: %s", err) - } - result.CseVersion = *cseVersion - - // Retrieve the Owner - if rde.DefinedEntity.Owner == nil { - return nil, fmt.Errorf("could not read Owner from RDE") - } - result.Owner = rde.DefinedEntity.Owner.Name - // Here we retrieve several items that we need from now onwards, like Storage Profiles and Compute Policies storageProfiles := map[string]string{} if rde.client.IsSysAdmin { @@ -343,76 +440,6 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu i++ } - // Add all events to the resulting cluster - for _, s := range capvcd.Status.VcdKe.EventSet { - result.Events = append(result.Events, CseClusterEvent{ - Name: s.Name, - Type: "event", - OccurredAt: s.OccurredAt, - Details: s.AdditionalDetails.DetailedEvent, - }) - } - for _, s := range capvcd.Status.VcdKe.ErrorSet { - result.Events = append(result.Events, CseClusterEvent{ - Name: s.Name, - Type: "error", - OccurredAt: s.OccurredAt, - Details: s.AdditionalDetails.DetailedError, - }) - } - for _, s := range capvcd.Status.Capvcd.EventSet { - result.Events = append(result.Events, CseClusterEvent{ - Name: s.Name, - Type: "event", - OccurredAt: s.OccurredAt, - Details: s.Name, - }) - } - for _, s := range capvcd.Status.Capvcd.ErrorSet { - result.Events = append(result.Events, CseClusterEvent{ - Name: s.Name, - Type: "error", - OccurredAt: s.OccurredAt, - Details: s.AdditionalDetails.DetailedError, - }) - } - for _, s := range capvcd.Status.Cpi.EventSet { - result.Events = append(result.Events, CseClusterEvent{ - Name: s.Name, - Type: "event", - OccurredAt: s.OccurredAt, - Details: s.Name, - }) - } - for _, s := range capvcd.Status.Cpi.ErrorSet { - result.Events = append(result.Events, CseClusterEvent{ - Name: s.Name, - Type: "error", - OccurredAt: s.OccurredAt, - Details: s.AdditionalDetails.DetailedError, - }) - } - for _, s := range capvcd.Status.Csi.EventSet { - result.Events = append(result.Events, CseClusterEvent{ - Name: s.Name, - Type: "event", - OccurredAt: s.OccurredAt, - Details: s.Name, - }) - } - for _, s := range capvcd.Status.Csi.ErrorSet { - result.Events = append(result.Events, CseClusterEvent{ - Name: s.Name, - Type: "error", - OccurredAt: s.OccurredAt, - Details: s.AdditionalDetails.DetailedError, - }) - } - - sort.SliceStable(result.Events, func(i, j int) bool { - return result.Events[i].OccurredAt.Before(result.Events[j].OccurredAt) - }) - return result, nil } @@ -767,20 +794,25 @@ func (tkgVersions tkgVersionBundle) compareTkgVersion(tkgVersion string) int { } func (tkgVersions tkgVersionBundle) kubernetesVersionIsOneMinorHigher(kubernetesVersion string) bool { - receiverVersionTokens := strings.Split(tkgVersions.KubernetesVersion, ".") - if len(receiverVersionTokens) < 3 { + receiverVersion, err := semver.NewVersion(tkgVersions.KubernetesVersion) + if err != nil { return false } - vTokens := strings.Split(kubernetesVersion, ".") - if len(vTokens) < 3 { + inputVersion, err := semver.NewVersion(kubernetesVersion) + if err != nil { return false } - minor, err := strconv.Atoi(receiverVersionTokens[1]) - if err != nil { + + receiverVersionSegments := receiverVersion.Segments() + if len(receiverVersionSegments) < 2 { + return false + } + inputSegments := inputVersion.Segments() + if len(inputSegments) < 2 { return false } - return receiverVersionTokens[0] == vTokens[0] && fmt.Sprintf("%d", minor+1) == vTokens[1] && receiverVersionTokens[2] == vTokens[2] + return receiverVersionSegments[0] == inputSegments[0] && receiverVersionSegments[1]-1 == inputSegments[1] } // getVcdKeConfig gets the required information from the CSE Server configuration RDE (VCDKEConfig), such as the diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index daa698284..6a28286b0 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -33,9 +33,6 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) // as well. // So in this special case this "optimization" would optimize nothing. The same happens with other YAML values. if input.KubernetesTemplateOvaId != nil { - if !cluster.Upgradeable { - return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the cluster is not upgradeable") - } vAppTemplate, err := getVAppTemplateById(cluster.client, *input.KubernetesTemplateOvaId) if err != nil { return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", *input.KubernetesTemplateOvaId, err) @@ -45,7 +42,7 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) if err != nil { return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the TKG versions of OVA '%s': %s", *input.KubernetesTemplateOvaId, err) } - if versions.compareTkgVersion(cluster.capvcdType.Status.Capvcd.Upgrade.Target.TkgVersion) != 1 || !versions.kubernetesVersionIsOneMinorHigher(cluster.capvcdType.Status.Capvcd.Upgrade.Target.KubernetesVersion) { + if versions.compareTkgVersion(cluster.capvcdType.Status.Capvcd.Upgrade.Current.TkgVersion) != 1 || !versions.kubernetesVersionIsOneMinorHigher(cluster.capvcdType.Status.Capvcd.Upgrade.Current.KubernetesVersion) { return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the new one '%s' has an older TKG/Kubernetes version (%s/%s)", vAppTemplate.VAppTemplate.Name, versions.TkgVersion, versions.KubernetesVersion) } err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate) diff --git a/types/v56/cse.go b/types/v56/cse.go index 790954fb4..21918baed 100644 --- a/types/v56/cse.go +++ b/types/v56/cse.go @@ -30,6 +30,7 @@ type Capvcd struct { Name string `json:"name,omitempty"` OccurredAt time.Time `json:"occurredAt,omitempty"` VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` AdditionalDetails struct { DetailedEvent string `json:"Detailed Event,omitempty"` } `json:"additionalDetails,omitempty"` @@ -50,6 +51,8 @@ type Capvcd struct { EventSet []struct { Name string `json:"name,omitempty"` OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` AdditionalDetails struct { DetailedDescription string `json:"Detailed Description,omitempty"` } `json:"additionalDetails,omitempty"` @@ -70,6 +73,7 @@ type Capvcd struct { Name string `json:"name,omitempty"` OccurredAt time.Time `json:"occurredAt,omitempty"` VcdResourceId string `json:"vcdResourceId,omitempty"` + VcdResourceName string `json:"vcdResourceName,omitempty"` AdditionalDetails struct { DetailedEvent string `json:"Detailed Event,omitempty"` } `json:"additionalDetails,omitempty"` @@ -112,10 +116,6 @@ type Capvcd struct { TkgVersion string `json:"tkgVersion,omitempty"` KubernetesVersion string `json:"kubernetesVersion,omitempty"` } `json:"current,omitempty"` - Target struct { - TkgVersion string `json:"tkgVersion,omitempty"` - KubernetesVersion string `json:"kubernetesVersion,omitempty"` - } `json:"target,omitempty"` } `json:"upgrade,omitempty"` EventSet []struct { Name string `json:"name,omitempty"` From f7a7fa6a30cfcd97d08040da9bad126364f9db9f Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 14 Feb 2024 13:48:21 +0100 Subject: [PATCH 045/115] Fixes Signed-off-by: abarreiro --- govcd/cse.go | 9 +++------ govcd/cse_test.go | 16 ++++------------ govcd/cse_type.go | 5 ----- govcd/cse_yaml.go | 18 ++++++++++++++++-- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index f0cb151ec..f33792219 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -142,12 +142,13 @@ func (cluster *CseKubernetesCluster) GetKubeconfig() (string, error) { Capvcd types.Capvcd `json:"entity,omitempty"` } result := invocationResult{} + err = rde.InvokeBehaviorAndMarshal(fmt.Sprintf("urn:vcloud:behavior-interface:getFullEntity:cse:capvcd:%s", versions.CseInterfaceVersion), types.BehaviorInvocation{}, &result) if err != nil { - return "", err + return "", fmt.Errorf("could not retrieve the Kubeconfig, the Behavior invocation failed: %s", err) } if result.Capvcd.Status.Capvcd.Private.KubeConfig == "" { - return "", fmt.Errorf("could not retrieve the Kubeconfig from the invocation of the Behavior") + return "", fmt.Errorf("could not retrieve the Kubeconfig, the Behavior invocation succeeded but the Kubeconfig is empty") } return result.Capvcd.Status.Capvcd.Private.KubeConfig, nil } @@ -269,10 +270,6 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh cluster.capvcdType.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors } - // Computed attributes that are required, such as the VcdKeConfig version - input.clusterName = cluster.Name - input.vcdKeConfigVersion = cluster.capvcdType.Status.VcdKe.VcdKeVersion - input.cseVersion = cluster.CseVersion updatedCapiYaml, err := cluster.updateCapiYaml(input) if err != nil { return err diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 085bf8ec1..df53073f4 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -220,22 +220,14 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:2d137956-e702-474e-a4df-7a51c868f22c") check.Assert(err, IsNil) - _, err = cluster.GetSupportedUpgrades(true) - check.Assert(err, IsNil) - // check.Assert(len(upgrades) > 0, Equals, true) - workerPoolName := "node-pool-1" - //kubeconfig, err := cluster.GetKubeconfig() - //check.Assert(err, IsNil) - //check.Assert(true, Equals, strings.Contains(kubeconfig, cluster.Name)) - //check.Assert(true, Equals, strings.Contains(kubeconfig, "client-certificate-data")) - //check.Assert(true, Equals, strings.Contains(kubeconfig, "certificate-authority-data")) - //check.Assert(true, Equals, strings.Contains(kubeconfig, "client-key-data")) + _, err = cluster.GetKubeconfig() + check.Assert(err, IsNil) // Perform the update - // err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, true) - // check.Assert(err, IsNil) + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, true) + check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up foundWorkerPool := false diff --git a/govcd/cse_type.go b/govcd/cse_type.go index 1be10ca60..e24bd5267 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -94,11 +94,6 @@ type CseClusterUpdateInput struct { NewWorkerPools *[]CseWorkerPoolSettings NodeHealthCheck *bool AutoRepairOnErrors *bool - - // Private fields that are computed, not requested to the consumer of this struct - vcdKeConfigVersion string - clusterName string - cseVersion semver.Version } // CseControlPlaneUpdateInput defines the required configuration that the Control Plane of the Container Service Extension (CSE) Kubernetes cluster diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 6a28286b0..595c60a23 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -51,6 +51,13 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) } } + if input.WorkerPools != nil { + err := cseUpdateWorkerPoolsInYaml(yamlDocs, *input.WorkerPools) + if err != nil { + return cluster.capvcdType.Spec.CapiYaml, err + } + } + if input.NewWorkerPools != nil { yamlDocs, err = cseAddWorkerPoolsInYaml(yamlDocs, *cluster, *input.NewWorkerPools) if err != nil { @@ -58,12 +65,19 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) } } + if input.ControlPlane != nil { + err := cseUpdateControlPlaneInYaml(yamlDocs, *input.ControlPlane) + if err != nil { + return cluster.capvcdType.Spec.CapiYaml, err + } + } + if input.NodeHealthCheck != nil { - vcdKeConfig, err := getVcdKeConfig(cluster.client, input.vcdKeConfigVersion, *input.NodeHealthCheck) + vcdKeConfig, err := getVcdKeConfig(cluster.client, cluster.capvcdType.Status.VcdKe.VcdKeVersion, *input.NodeHealthCheck) if err != nil { return "", err } - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, input.clusterName, input.cseVersion, vcdKeConfig) + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, cluster.Name, cluster.CseVersion, vcdKeConfig) if err != nil { return "", err } From 382a8d12587596dd81f6dd21cb005303c7e6a1a2 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 14 Feb 2024 14:06:20 +0100 Subject: [PATCH 046/115] Attempt to fix Kubeconfig override Signed-off-by: abarreiro --- govcd/cse.go | 3 +++ types/v56/cse.go | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index f33792219..516e10915 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -147,6 +147,9 @@ func (cluster *CseKubernetesCluster) GetKubeconfig() (string, error) { if err != nil { return "", fmt.Errorf("could not retrieve the Kubeconfig, the Behavior invocation failed: %s", err) } + if result.Capvcd.Status.Capvcd.Private == nil { + return "", fmt.Errorf("could not retrieve the Kubeconfig, the Behavior invocation succeeded but the Kubeconfig is nil") + } if result.Capvcd.Status.Capvcd.Private.KubeConfig == "" { return "", fmt.Errorf("could not retrieve the Kubeconfig, the Behavior invocation succeeded but the Kubeconfig is empty") } diff --git a/types/v56/cse.go b/types/v56/cse.go index 21918baed..619fe5ce3 100644 --- a/types/v56/cse.go +++ b/types/v56/cse.go @@ -8,7 +8,9 @@ type Capvcd struct { Kind string `json:"kind,omitempty"` Spec struct { VcdKe struct { - Secure struct { + // NOTE: "Secure" struct needs to be a pointer to avoid overriding with empty values by mistake, as VCD doesn't return RDE fields + // marked with "x-vcloud-restricted: secure" + Secure *struct { ApiToken string `json:"apiToken,omitempty"` } `json:"secure,omitempty"` IsVCDKECluster bool `json:"isVCDKECluster,omitempty"` @@ -105,9 +107,11 @@ type Capvcd struct { } `json:"defaultStorageClass,omitempty"` } `json:"vcdKe,omitempty"` Capvcd struct { - Uid string `json:"uid,omitempty"` - Phase string `json:"phase,omitempty"` - Private struct { + Uid string `json:"uid,omitempty"` + Phase string `json:"phase,omitempty"` + // NOTE: "Private" struct needs to be a pointer to avoid overriding with empty values by mistake, as VCD doesn't return RDE fields + // marked with "x-vcloud-restricted: secure" + Private *struct { KubeConfig string `json:"kubeConfig,omitempty"` } `json:"private,omitempty"` Upgrade struct { From 3a97f3e175affac86a8c8a7eebad6441c7154f8b Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 14 Feb 2024 14:21:21 +0100 Subject: [PATCH 047/115] Worker pools unique names Signed-off-by: abarreiro --- govcd/cse_test.go | 26 +++++++++++++++++--------- govcd/cse_util.go | 5 +++++ govcd/cse_yaml.go | 8 ++++++++ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index df53073f4..fbf4cc784 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -240,20 +240,28 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { check.Assert(foundWorkerPool, Equals, true) // Revert back (resources can be limited) - // err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 1}}, true) - // check.Assert(err, IsNil) + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 1}}, true) + check.Assert(err, IsNil) // Perform the update - // err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 2}, true) - // check.Assert(err, IsNil) + err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 2}, true) + check.Assert(err, IsNil) // Post-check. This should be 2, as it should have scaled up - // check.Assert(cluster.ControlPlane.MachineCount, Equals, 2) + check.Assert(cluster.ControlPlane.MachineCount, Equals, 2) // Revert back (resources can be limited) - // err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) - // check.Assert(err, IsNil) + err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) + check.Assert(err, IsNil) - // err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) - // check.Assert(err, IsNil) + err = cluster.AddWorkerPools([]CseWorkerPoolSettings{{ + Name: "node-pool-2", + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyId: cluster.WorkerPools[0].SizingPolicyId, + PlacementPolicyId: "", + VGpuPolicyId: "", + StorageProfileId: cluster.WorkerPools[0].StorageProfileId, + }}, true) + check.Assert(err, IsNil) } diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 4e78b5c1c..8bb943044 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -533,7 +533,11 @@ func (input *CseClusterSettings) validate() error { if len(input.WorkerPools) == 0 { return fmt.Errorf("there must be at least one Worker Pool") } + existingWorkerPools := map[string]bool{} for _, workerPool := range input.WorkerPools { + if _, alreadyExists := existingWorkerPools[workerPool.Name]; alreadyExists { + return fmt.Errorf("the names of the Worker Pools must be unique, but '%s' is repeated", workerPool.Name) + } if workerPool.MachineCount < 1 { return fmt.Errorf("number of Worker Pool '%s' nodes must higher than 0, but it was '%d'", workerPool.Name, workerPool.MachineCount) } @@ -543,6 +547,7 @@ func (input *CseClusterSettings) validate() error { if !cseNamesRegex.MatchString(workerPool.Name) { return fmt.Errorf("the Worker Pool name '%s' must contain only lowercase alphanumeric characters or '-', start with an alphabetic character, end with an alphanumeric, and contain at most 31 characters", workerPool.Name) } + existingWorkerPools[workerPool.Name] = true } if input.DefaultStorageClass != nil { // This field is optional if !cseNamesRegex.MatchString(input.DefaultStorageClass.Name) { diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 595c60a23..82fcb317d 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -52,6 +52,14 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) } if input.WorkerPools != nil { + // Worker pool names must be unique + for _, existingPool := range cluster.WorkerPools { + for _, newPool := range *input.NewWorkerPools { + if newPool.Name == existingPool.Name { + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("there is an existing Worker Pool with name %s", existingPool.Name) + } + } + } err := cseUpdateWorkerPoolsInYaml(yamlDocs, *input.WorkerPools) if err != nil { return cluster.capvcdType.Spec.CapiYaml, err From 7ca3fedee5b8a25c5e3bed196d137ab8f8bc07a9 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 14 Feb 2024 14:36:25 +0100 Subject: [PATCH 048/115] # Signed-off-by: abarreiro --- govcd/cse_yaml.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 82fcb317d..696c92efd 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -16,7 +16,7 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) if cluster == nil { return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("receiver cluster is nil") } - if input.ControlPlane == nil && input.WorkerPools == nil && input.NodeHealthCheck == nil && input.KubernetesTemplateOvaId == nil { + if input.ControlPlane == nil && input.WorkerPools == nil && input.NodeHealthCheck == nil && input.KubernetesTemplateOvaId == nil && input.NewWorkerPools == nil { return cluster.capvcdType.Spec.CapiYaml, nil } @@ -52,21 +52,22 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) } if input.WorkerPools != nil { + err := cseUpdateWorkerPoolsInYaml(yamlDocs, *input.WorkerPools) + if err != nil { + return cluster.capvcdType.Spec.CapiYaml, err + } + } + + if input.NewWorkerPools != nil { // Worker pool names must be unique for _, existingPool := range cluster.WorkerPools { for _, newPool := range *input.NewWorkerPools { if newPool.Name == existingPool.Name { - return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("there is an existing Worker Pool with name %s", existingPool.Name) + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("there is an existing Worker Pool with name '%s'", existingPool.Name) } } } - err := cseUpdateWorkerPoolsInYaml(yamlDocs, *input.WorkerPools) - if err != nil { - return cluster.capvcdType.Spec.CapiYaml, err - } - } - if input.NewWorkerPools != nil { yamlDocs, err = cseAddWorkerPoolsInYaml(yamlDocs, *cluster, *input.NewWorkerPools) if err != nil { return cluster.capvcdType.Spec.CapiYaml, err @@ -211,6 +212,8 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]interface{}, workerPo // described by the input parameters. // NOTE: This function doesn't modify the input, but returns a copy of the YAML with the added unmarshalled documents. func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernetesCluster, newWorkerPools []CseWorkerPoolSettings) ([]map[string]interface{}, error) { + + // FIXME: Convert IDs -> Names internalSettings := cseClusterSettingsInternal{WorkerPools: make([]cseWorkerPoolSettingsInternal, len(newWorkerPools))} for i, workerPool := range newWorkerPools { internalSettings.WorkerPools[i] = cseWorkerPoolSettingsInternal{ From 57560c7cbb0c261d9db427bed1908d89b284a503 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 14 Feb 2024 14:49:30 +0100 Subject: [PATCH 049/115] Refactor Signed-off-by: abarreiro --- govcd/cse_util.go | 58 +++++++++++++++++++++++++++-------------------- govcd/cse_yaml.go | 24 ++++++++++++++++---- 2 files changed, 52 insertions(+), 30 deletions(-) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 8bb943044..bca92f630 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -633,12 +633,6 @@ func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClus } output.RdeType = rdeType.DefinedEntityType - // The input to create a cluster uses different entities IDs, but CSE cluster creation process uses Names. - // For that reason, we need to transform IDs to Names by querying VCD. This process is optimized with a tiny cache map. - idToNameCache := map[string]string{ - "": "", // Default empty value to map optional values that were not set, to avoid extra checks. For example, an empty vGPU Policy. - } - // Gather all the IDs of the Compute Policies and Storage Profiles, so we can transform them to Names in bulk. var computePolicyIds []string var storageProfileIds []string @@ -649,25 +643,9 @@ func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClus computePolicyIds = append(computePolicyIds, input.ControlPlane.SizingPolicyId, input.ControlPlane.PlacementPolicyId) storageProfileIds = append(storageProfileIds, input.ControlPlane.StorageProfileId, input.DefaultStorageClass.StorageProfileId) - // Retrieve the Compute Policies and Storage Profiles names and put them in the cache. The cache - // reduces the calls to VCD. The URN format used by VCD guarantees that IDs are unique, so there is no possibility of clashes here. - for _, id := range storageProfileIds { - if _, alreadyPresent := idToNameCache[id]; !alreadyPresent { - storageProfile, err := getStorageProfileById(org.client, id) - if err != nil { - return nil, fmt.Errorf("could not retrieve Storage Profile with ID '%s': %s", id, err) - } - idToNameCache[id] = storageProfile.Name - } - } - for _, id := range computePolicyIds { - if _, alreadyPresent := idToNameCache[id]; !alreadyPresent { - computePolicy, err := getVdcComputePolicyV2ById(org.client, id) - if err != nil { - return nil, fmt.Errorf("could not retrieve Compute Policy with ID '%s': %s", id, err) - } - idToNameCache[id] = computePolicy.VdcComputePolicyV2.Name - } + idToNameCache, err := idToNames(org.client, computePolicyIds, storageProfileIds) + if err != nil { + return nil, err } // Now that everything is cached in memory, we can build the Node pools and Storage Class payloads in a trivial way. @@ -859,6 +837,36 @@ func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheck return result, nil } +// idToNames returns a map that associates Compute Policies/Storage Profiles IDs with their respective names. +// This is useful as the input to create/update a cluster uses different entities IDs, but CSE cluster creation/update process uses Names. +// For that reason, we need to transform IDs to Names by querying VCD +func idToNames(client *Client, computePolicyIds, storageProfileIds []string) (map[string]string, error) { + result := map[string]string{ + "": "", // Default empty value to map optional values that were not set, to avoid extra checks. For example, an empty vGPU Policy. + } + // Retrieve the Compute Policies and Storage Profiles names and put them in the cache. The cache + // reduces the calls to VCD. The URN format used by VCD guarantees that IDs are unique, so there is no possibility of clashes here. + for _, id := range storageProfileIds { + if _, alreadyPresent := result[id]; !alreadyPresent { + storageProfile, err := getStorageProfileById(client, id) + if err != nil { + return nil, fmt.Errorf("could not retrieve Storage Profile with ID '%s': %s", id, err) + } + result[id] = storageProfile.Name + } + } + for _, id := range computePolicyIds { + if _, alreadyPresent := result[id]; !alreadyPresent { + computePolicy, err := getVdcComputePolicyV2ById(client, id) + if err != nil { + return nil, fmt.Errorf("could not retrieve Compute Policy with ID '%s': %s", id, err) + } + result[id] = computePolicy.VdcComputePolicyV2.Name + } + } + return result, nil +} + // getCseTemplate reads the Go template present in the embedded cseFiles filesystem. func getCseTemplate(cseVersion semver.Version, templateName string) (string, error) { minimumVersion, err := semver.NewVersion("4.1") diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 696c92efd..d5864ecc4 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -212,18 +212,32 @@ func cseUpdateWorkerPoolsInYaml(yamlDocuments []map[string]interface{}, workerPo // described by the input parameters. // NOTE: This function doesn't modify the input, but returns a copy of the YAML with the added unmarshalled documents. func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernetesCluster, newWorkerPools []CseWorkerPoolSettings) ([]map[string]interface{}, error) { + if len(newWorkerPools) == 0 { + return docs, nil + } + + var computePolicyIds []string + var storageProfileIds []string + for _, w := range newWorkerPools { + computePolicyIds = append(computePolicyIds, w.SizingPolicyId, w.PlacementPolicyId, w.VGpuPolicyId) + storageProfileIds = append(storageProfileIds, w.StorageProfileId) + } + + idToNameCache, err := idToNames(cluster.client, computePolicyIds, storageProfileIds) + if err != nil { + return nil, err + } - // FIXME: Convert IDs -> Names internalSettings := cseClusterSettingsInternal{WorkerPools: make([]cseWorkerPoolSettingsInternal, len(newWorkerPools))} for i, workerPool := range newWorkerPools { internalSettings.WorkerPools[i] = cseWorkerPoolSettingsInternal{ Name: workerPool.Name, MachineCount: workerPool.MachineCount, DiskSizeGi: workerPool.DiskSizeGi, - SizingPolicyName: workerPool.SizingPolicyId, - PlacementPolicyName: workerPool.PlacementPolicyId, - VGpuPolicyName: workerPool.VGpuPolicyId, - StorageProfileName: workerPool.StorageProfileId, + StorageProfileName: idToNameCache[workerPool.StorageProfileId], + SizingPolicyName: idToNameCache[workerPool.SizingPolicyId], + VGpuPolicyName: idToNameCache[workerPool.VGpuPolicyId], + PlacementPolicyName: idToNameCache[workerPool.PlacementPolicyId], } } internalSettings.Name = cluster.Name From c33dab3252d50e5129da195b5917bc5ca80ec8c2 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 14 Feb 2024 14:50:17 +0100 Subject: [PATCH 050/115] Refactor Signed-off-by: abarreiro --- govcd/cse_util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index bca92f630..a100deec2 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -844,8 +844,8 @@ func idToNames(client *Client, computePolicyIds, storageProfileIds []string) (ma result := map[string]string{ "": "", // Default empty value to map optional values that were not set, to avoid extra checks. For example, an empty vGPU Policy. } - // Retrieve the Compute Policies and Storage Profiles names and put them in the cache. The cache - // reduces the calls to VCD. The URN format used by VCD guarantees that IDs are unique, so there is no possibility of clashes here. + // Retrieve the Compute Policies and Storage Profiles names and put them in the resulting map. This map also can + // be used to reduce the calls to VCD. The URN format used by VCD guarantees that IDs are unique, so there is no possibility of clashes here. for _, id := range storageProfileIds { if _, alreadyPresent := result[id]; !alreadyPresent { storageProfile, err := getStorageProfileById(client, id) From 73c7ac591415c97b95d940a30889452cf50cf59f Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 14 Feb 2024 17:09:59 +0100 Subject: [PATCH 051/115] Fixes Signed-off-by: abarreiro --- govcd/cse_template.go | 11 +++--- govcd/cse_test.go | 6 ++-- govcd/cse_util_unit_test.go | 4 +-- govcd/cse_yaml.go | 60 ++++++++++++++++++++----------- govcd/cse_yaml_unit_test.go | 72 ------------------------------------- 5 files changed, 51 insertions(+), 102 deletions(-) diff --git a/govcd/cse_template.go b/govcd/cse_template.go index 29a6e8497..62c2a6fc5 100644 --- a/govcd/cse_template.go +++ b/govcd/cse_template.go @@ -78,7 +78,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateNodePoolYaml() (strin buf := &bytes.Buffer{} // We can have many worker pools, we build a YAML object for each one of them. - for _, workerPool := range clusterSettings.WorkerPools { + for i, workerPool := range clusterSettings.WorkerPools { // Check the correctness of the compute policies in the node pool block if workerPool.PlacementPolicyName != "" && workerPool.VGpuPolicyName != "" { @@ -105,7 +105,10 @@ func (clusterSettings *cseClusterSettingsInternal) generateNodePoolYaml() (strin }); err != nil { return "", fmt.Errorf("could not generate a correct Node Pool YAML: %s", err) } - resultYaml += fmt.Sprintf("%s\n---\n", buf.String()) + resultYaml += fmt.Sprintf("%s\n", buf.String()) + if i < len(clusterSettings.WorkerPools)-1 { + resultYaml += "---\n" + } buf.Reset() } return resultYaml, nil @@ -140,7 +143,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateMemoryHealthCheckYaml }); err != nil { return "", fmt.Errorf("could not generate a correct Memory Health Check YAML: %s", err) } - return fmt.Sprintf("%s\n---\n", buf.String()), nil + return fmt.Sprintf("%s\n", buf.String()), nil } @@ -205,7 +208,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( return "", fmt.Errorf("could not generate a correct CAPI YAML: %s", err) } // The final "pretty" YAML. To embed it in the final payload it must be marshaled into a one-line JSON string - prettyYaml := fmt.Sprintf("%s\n%s\n%s", memoryHealthCheckYaml, nodePoolYaml, buf.String()) + prettyYaml := fmt.Sprintf("%s\n---\n%s\n---\n%s", memoryHealthCheckYaml, nodePoolYaml, buf.String()) // We don't use a standard json.Marshal() as the YAML contains special // characters that are not encoded properly, such as '<'. diff --git a/govcd/cse_test.go b/govcd/cse_test.go index fbf4cc784..61168ab8d 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -217,10 +217,10 @@ func (vcd *TestVCD) Test_Cse(check *C) { } func (vcd *TestVCD) Test_Deleteme(check *C) { - cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:2d137956-e702-474e-a4df-7a51c868f22c") + cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:fd8e63dc-9127-407f-bf7b-29357442b8b4") check.Assert(err, IsNil) - workerPoolName := "node-pool-1" + workerPoolName := "test2-worker-node-pool-1" _, err = cluster.GetKubeconfig() check.Assert(err, IsNil) @@ -255,7 +255,7 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { check.Assert(err, IsNil) err = cluster.AddWorkerPools([]CseWorkerPoolSettings{{ - Name: "node-pool-2", + Name: "node-pool-5", MachineCount: 1, DiskSizeGi: 20, SizingPolicyId: cluster.WorkerPools[0].SizingPolicyId, diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index d76e41287..1d9139eea 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -82,8 +82,8 @@ func Test_getTkgVersionBundleFromVAppTemplate(t *testing.T) { ProductSection: &types.ProductSection{ Property: []*types.Property{ { - Key: "VERSION", - Value: &types.Value{Value: "v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc"}, + Key: "VERSION", + DefaultValue: "v1.25.7+vmware.2-tkg.1-8a74b9f12e488c54605b3537acb683bc", }, }, }, diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index d5864ecc4..7927fd251 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -27,25 +27,8 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("error unmarshaling YAML: %s", err) } - // As a side note, we can't optimize this one with "if equals do nothing" because - // in order to retrieve the current value we would need to explore the YAML anyway, which is what we also need to do to update it. - // Also, even if we did it, the current value obtained from YAML would be a Name, but the new value is an ID, so we would need to query VCD anyway - // as well. - // So in this special case this "optimization" would optimize nothing. The same happens with other YAML values. - if input.KubernetesTemplateOvaId != nil { - vAppTemplate, err := getVAppTemplateById(cluster.client, *input.KubernetesTemplateOvaId) - if err != nil { - return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", *input.KubernetesTemplateOvaId, err) - } - // Check the versions of the selected OVA before upgrading - versions, err := getTkgVersionBundleFromVAppTemplate(vAppTemplate.VAppTemplate) - if err != nil { - return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the TKG versions of OVA '%s': %s", *input.KubernetesTemplateOvaId, err) - } - if versions.compareTkgVersion(cluster.capvcdType.Status.Capvcd.Upgrade.Current.TkgVersion) != 1 || !versions.kubernetesVersionIsOneMinorHigher(cluster.capvcdType.Status.Capvcd.Upgrade.Current.KubernetesVersion) { - return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the new one '%s' has an older TKG/Kubernetes version (%s/%s)", vAppTemplate.VAppTemplate.Name, versions.TkgVersion, versions.KubernetesVersion) - } - err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate) + if input.ControlPlane != nil { + err := cseUpdateControlPlaneInYaml(yamlDocs, *input.ControlPlane) if err != nil { return cluster.capvcdType.Spec.CapiYaml, err } @@ -58,6 +41,7 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) } } + // Order matters. We need to add the new pools before updating the Kubernetes template. if input.NewWorkerPools != nil { // Worker pool names must be unique for _, existingPool := range cluster.WorkerPools { @@ -74,8 +58,25 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) } } - if input.ControlPlane != nil { - err := cseUpdateControlPlaneInYaml(yamlDocs, *input.ControlPlane) + // As a side note, we can't optimize this one with "if equals do nothing" because + // in order to retrieve the current value we would need to explore the YAML anyway, which is what we also need to do to update it. + // Also, even if we did it, the current value obtained from YAML would be a Name, but the new value is an ID, so we would need to query VCD anyway + // as well. + // So in this special case this "optimization" would optimize nothing. The same happens with other YAML values. + if input.KubernetesTemplateOvaId != nil { + vAppTemplate, err := getVAppTemplateById(cluster.client, *input.KubernetesTemplateOvaId) + if err != nil { + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the Kubernetes Template OVA with ID '%s': %s", *input.KubernetesTemplateOvaId, err) + } + // Check the versions of the selected OVA before upgrading + versions, err := getTkgVersionBundleFromVAppTemplate(vAppTemplate.VAppTemplate) + if err != nil { + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("could not retrieve the TKG versions of OVA '%s': %s", *input.KubernetesTemplateOvaId, err) + } + if versions.compareTkgVersion(cluster.capvcdType.Status.Capvcd.Upgrade.Current.TkgVersion) != 1 || !versions.kubernetesVersionIsOneMinorHigher(cluster.capvcdType.Status.Capvcd.Upgrade.Current.KubernetesVersion) { + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("cannot perform an OVA change as the new one '%s' has an older TKG/Kubernetes version (%s/%s)", vAppTemplate.VAppTemplate.Name, versions.TkgVersion, versions.KubernetesVersion) + } + err = cseUpdateKubernetesTemplateInYaml(yamlDocs, vAppTemplate.VAppTemplate) if err != nil { return cluster.capvcdType.Spec.CapiYaml, err } @@ -240,6 +241,23 @@ func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernete PlacementPolicyName: idToNameCache[workerPool.PlacementPolicyId], } } + + // Extra information needed to render the YAML. As all the worker pools share the same + // Kubernetes OVA name, version and Catalog, we pick this info from any of the available ones. + for _, doc := range docs { + if internalSettings.CatalogName == "" && doc["kind"] == "VCDMachineTemplate" { + internalSettings.CatalogName = traverseMapAndGet[string](doc, "spec.template.spec.catalog") + } + if internalSettings.KubernetesTemplateOvaName == "" && doc["kind"] == "VCDMachineTemplate" { + internalSettings.KubernetesTemplateOvaName = traverseMapAndGet[string](doc, "spec.template.spec.template") + } + if internalSettings.TkgVersionBundle.KubernetesVersion == "" && doc["kind"] == "MachineDeployment" { + internalSettings.TkgVersionBundle.KubernetesVersion = traverseMapAndGet[string](doc, "spec.template.spec.version") + } + if internalSettings.CatalogName != "" && internalSettings.KubernetesTemplateOvaName != "" && internalSettings.TkgVersionBundle.KubernetesVersion != "" { + break + } + } internalSettings.Name = cluster.Name internalSettings.CseVersion = cluster.CseVersion nodePoolsYaml, err := internalSettings.generateNodePoolYaml() diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index be65bfc99..677145960 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -180,78 +180,6 @@ func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { } } -// Test_cseAddWorkerPoolsInYaml tests the addition process of the Worker pools in a CAPI YAML. -func Test_cseAddWorkerPoolsInYaml(t *testing.T) { - version, err := semver.NewVersion("4.1") - if err != nil { - t.Fatalf("could not create version: %s", err) - } - capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") - if err != nil { - t.Fatalf("could not read CAPI YAML test file: %s", err) - } - - yamlDocs, err := unmarshalMultipleYamlDocuments(string(capiYaml)) - if err != nil { - t.Fatalf("could not unmarshal CAPI YAML test file: %s", err) - } - - // The worker pools should have now the new details updated - poolCount := 0 - for _, document := range yamlDocs { - if document["kind"] != "MachineDeployment" { - continue - } - poolCount++ - } - - // We call the function to update the old pools with the new ones - newNodePools := []CseWorkerPoolSettings{{ - Name: "new-pool", - MachineCount: 35, - DiskSizeGi: 20, - SizingPolicyId: "dummy", - PlacementPolicyId: "", - VGpuPolicyId: "", - StorageProfileId: "*", - }} - - newYamlDocs, err := cseAddWorkerPoolsInYaml(yamlDocs, CseKubernetesCluster{ - CseClusterSettings: CseClusterSettings{ - CseVersion: *version, - Name: "dummy", - }, - }, newNodePools) - if err != nil { - t.Fatalf("%s", err) - } - - // The worker pools should have now the new details updated - var newPool map[string]interface{} - newPoolCount := 0 - for _, document := range newYamlDocs { - if document["kind"] != "MachineDeployment" { - continue - } - - name := traverseMapAndGet[string](document, "metadata.name") - if name == "new-pool" { - newPool = document - } - newPoolCount++ - } - if newPool == nil { - t.Fatalf("should have found the new Worker Pool") - } - if poolCount != newPoolCount-1 { - t.Fatalf("should have one extra Worker Pool") - } - replicas := traverseMapAndGet[float64](newPool, "spec.replicas") - if replicas != 35 { - t.Fatalf("incorrect replicas: %.f", replicas) - } -} - // Test_cseUpdateControlPlaneInYaml tests the update process of the Control Plane in a CAPI YAML. func Test_cseUpdateControlPlaneInYaml(t *testing.T) { capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") From 8a03156b615ea359bfee6da87a83617dbc3997f1 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 14 Feb 2024 18:07:22 +0100 Subject: [PATCH 052/115] Refactor Signed-off-by: abarreiro --- govcd/cse.go | 6 +- govcd/{cse_template.go => cse_internal.go} | 204 +++++++++--------- ...unit_test.go => cse_internal_unit_test.go} | 11 +- govcd/cse_test.go | 43 +--- govcd/cse_type.go | 36 ++-- govcd/cse_util.go | 8 +- govcd/cse_yaml.go | 4 +- 7 files changed, 140 insertions(+), 172 deletions(-) rename govcd/{cse_template.go => cse_internal.go} (73%) rename govcd/{cse_template_unit_test.go => cse_internal_unit_test.go} (94%) diff --git a/govcd/cse.go b/govcd/cse.go index 516e10915..cd24a2e2f 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -46,7 +46,7 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) return "", fmt.Errorf("error creating the CSE Kubernetes cluster: %s", err) } - payload, err := internalSettings.getKubernetesClusterCreationPayload() + payload, err := internalSettings.getUnmarshaledRdePayload() if err != nil { return "", err } @@ -184,8 +184,7 @@ func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpd // As retrieving all OVAs one by one from VCD is expensive, the first time this method is called the returned OVAs are // cached to avoid querying VCD again multiple times. // If refreshOvas=true, this cache is cleared out and this method will query VCD for every vApp Template again. -// Therefore, the refreshOvas flag should be set to true only when VCD has new OVAs that need to be considered or the cluster -// has significantly changed since the first call. +// Therefore, the refreshOvas flag should be set to true only when VCD has new OVAs that need to be considered. func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refreshOvas bool) ([]*types.VAppTemplate, error) { if refreshOvas { cluster.supportedUpgrades = nil @@ -237,6 +236,7 @@ func (cluster *CseKubernetesCluster) SetHealthCheck(healthCheckEnabled bool, ref // SetAutoRepairOnErrors executes an update on the receiver cluster to change the flag that controls the auto-repair // capabilities of CSE. If refresh=true, it retrieves the latest state of the cluster from VCD before updating. +// NOTE: This can only be used in CSE versions < 4.1.1 func (cluster *CseKubernetesCluster) SetAutoRepairOnErrors(autoRepairOnErrors bool, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ AutoRepairOnErrors: &autoRepairOnErrors, diff --git a/govcd/cse_template.go b/govcd/cse_internal.go similarity index 73% rename from govcd/cse_template.go rename to govcd/cse_internal.go index 62c2a6fc5..c5e0408ee 100644 --- a/govcd/cse_template.go +++ b/govcd/cse_internal.go @@ -2,7 +2,7 @@ package govcd import ( "bytes" - _ "embed" + "embed" "encoding/base64" "encoding/json" "fmt" @@ -11,10 +11,15 @@ import ( "text/template" ) -// getKubernetesClusterCreationPayload gets the payload for the RDE that will trigger a Kubernetes cluster creation. -// It generates a valid YAML that is embedded inside the RDE JSON, then it is returned as an unmarshaled -// generic map, that allows to be sent to VCD as it is. -func (clusterSettings *cseClusterSettingsInternal) getKubernetesClusterCreationPayload() (map[string]interface{}, error) { +// This collection of files contains all the Go Templates and resources required for the Container Service Extension (CSE) methods +// to work. +// +//go:embed cse +var cseFiles embed.FS + +// getUnmarshaledRdePayload gets the unmarshaled JSON payload to create the Runtime Defined Entity that represents +// a CSE Kubernetes cluster, by using the receiver information. This method uses all the Go Templates stored in cseFiles +func (clusterSettings *cseClusterSettingsInternal) getUnmarshaledRdePayload() (map[string]interface{}, error) { if clusterSettings == nil { return nil, fmt.Errorf("the receiver cluster settings is nil") } @@ -62,94 +67,8 @@ func (clusterSettings *cseClusterSettingsInternal) getKubernetesClusterCreationP return result.(map[string]interface{}), nil } -// generateNodePoolYaml generates YAML blocks corresponding to the Kubernetes node pools. -func (clusterSettings *cseClusterSettingsInternal) generateNodePoolYaml() (string, error) { - if clusterSettings == nil { - return "", fmt.Errorf("the receiver cluster settings is nil") - } - - workerPoolTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_workerpool") - if err != nil { - return "", err - } - - nodePoolEmptyTmpl := template.Must(template.New(clusterSettings.Name + "-worker-pool").Parse(workerPoolTmpl)) - resultYaml := "" - buf := &bytes.Buffer{} - - // We can have many worker pools, we build a YAML object for each one of them. - for i, workerPool := range clusterSettings.WorkerPools { - - // Check the correctness of the compute policies in the node pool block - if workerPool.PlacementPolicyName != "" && workerPool.VGpuPolicyName != "" { - return "", fmt.Errorf("the worker pool '%s' should have either a Placement Policy or a vGPU Policy, not both", workerPool.Name) - } - placementPolicy := workerPool.PlacementPolicyName - if workerPool.VGpuPolicyName != "" { - placementPolicy = workerPool.VGpuPolicyName // For convenience, we just use one of the variables as both cannot be set at same time - } - - if err := nodePoolEmptyTmpl.Execute(buf, map[string]string{ - "ClusterName": clusterSettings.Name, - "NodePoolName": workerPool.Name, - "TargetNamespace": clusterSettings.Name + "-ns", - "Catalog": clusterSettings.CatalogName, - "VAppTemplate": clusterSettings.KubernetesTemplateOvaName, - "NodePoolSizingPolicy": workerPool.SizingPolicyName, - "NodePoolPlacementPolicy": placementPolicy, // Can be either Placement or vGPU policy - "NodePoolStorageProfile": workerPool.StorageProfileName, - "NodePoolDiskSize": fmt.Sprintf("%dGi", workerPool.DiskSizeGi), - "NodePoolEnableGpu": strconv.FormatBool(workerPool.VGpuPolicyName != ""), - "NodePoolMachineCount": strconv.Itoa(workerPool.MachineCount), - "KubernetesVersion": clusterSettings.TkgVersionBundle.KubernetesVersion, - }); err != nil { - return "", fmt.Errorf("could not generate a correct Node Pool YAML: %s", err) - } - resultYaml += fmt.Sprintf("%s\n", buf.String()) - if i < len(clusterSettings.WorkerPools)-1 { - resultYaml += "---\n" - } - buf.Reset() - } - return resultYaml, nil -} - -// generateMemoryHealthCheckYaml generates a YAML block corresponding to the Kubernetes memory health check. -func (clusterSettings *cseClusterSettingsInternal) generateMemoryHealthCheckYaml() (string, error) { - if clusterSettings == nil { - return "", fmt.Errorf("the receiver cluster settings is nil") - } - - if clusterSettings.VcdKeConfig.NodeStartupTimeout == "" && clusterSettings.VcdKeConfig.NodeUnknownTimeout == "" && clusterSettings.VcdKeConfig.NodeNotReadyTimeout == "" && - clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage == 0 { - return "", nil - } - - mhcTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_mhc") - if err != nil { - return "", err - } - - mhcEmptyTmpl := template.Must(template.New(clusterSettings.Name + "-mhc").Parse(mhcTmpl)) - buf := &bytes.Buffer{} - - if err := mhcEmptyTmpl.Execute(buf, map[string]string{ - "ClusterName": clusterSettings.Name, - "TargetNamespace": clusterSettings.Name + "-ns", - "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage), // With the 'percentage' suffix - "NodeStartupTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeStartupTimeout, "s", "")), // We assure don't duplicate the 's' suffix - "NodeUnknownTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeUnknownTimeout, "s", "")), // We assure don't duplicate the 's' suffix - "NodeNotReadyTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeNotReadyTimeout, "s", "")), // We assure don't duplicate the 's' suffix - }); err != nil { - return "", fmt.Errorf("could not generate a correct Memory Health Check YAML: %s", err) - } - return fmt.Sprintf("%s\n", buf.String()), nil - -} - -// generateCapiYamlAsJsonString generates the YAML string that is required during Kubernetes cluster creation, to be embedded -// in the CAPVCD cluster JSON payload. This function picks data from the Terraform schema and the createClusterDto to -// populate several Go templates and build a final YAML. +// generateCapiYamlAsJsonString generates the "capiYaml" property of the RDE that represents a Kubernetes cluster. This +// "capiYaml" property is a YAML encoded as a JSON string. This method uses the Go Templates stored in cseFiles. func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString() (string, error) { if clusterSettings == nil { return "", fmt.Errorf("the receiver cluster settings is nil") @@ -164,12 +83,12 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( sanitizedTemplate := strings.NewReplacer("%", "%%").Replace(clusterTmpl) capiYamlEmpty := template.Must(template.New(clusterSettings.Name + "-cluster").Parse(sanitizedTemplate)) - nodePoolYaml, err := clusterSettings.generateNodePoolYaml() + nodePoolYaml, err := clusterSettings.generateWorkerPoolsYaml() if err != nil { return "", err } - memoryHealthCheckYaml, err := clusterSettings.generateMemoryHealthCheckYaml() + memoryHealthCheckYaml, err := clusterSettings.generateMachineHealthCheckYaml() if err != nil { return "", err } @@ -210,8 +129,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( // The final "pretty" YAML. To embed it in the final payload it must be marshaled into a one-line JSON string prettyYaml := fmt.Sprintf("%s\n---\n%s\n---\n%s", memoryHealthCheckYaml, nodePoolYaml, buf.String()) - // We don't use a standard json.Marshal() as the YAML contains special - // characters that are not encoded properly, such as '<'. + // We don't use a standard json.Marshal() as the YAML contains special characters that are not encoded properly, such as '<'. buf.Reset() enc := json.NewEncoder(buf) enc.SetEscapeHTML(false) @@ -223,3 +141,95 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( // Removes trailing quotes from the final JSON string return strings.Trim(strings.TrimSpace(buf.String()), "\""), nil } + +// generateWorkerPoolsYaml generates YAML blocks corresponding to the cluster Worker Pools. The blocks are separated by +// the standard YAML separator (---), but does not add one at the end. +func (clusterSettings *cseClusterSettingsInternal) generateWorkerPoolsYaml() (string, error) { + if clusterSettings == nil { + return "", fmt.Errorf("the receiver cluster settings is nil") + } + + workerPoolTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_workerpool") + if err != nil { + return "", err + } + + nodePoolEmptyTmpl := template.Must(template.New(clusterSettings.Name + "-worker-pool").Parse(workerPoolTmpl)) + resultYaml := "" + buf := &bytes.Buffer{} + + // We can have many Worker Pools, we build a YAML object for each one of them. + for i, workerPool := range clusterSettings.WorkerPools { + + // Check the correctness of the Compute Policies in the node pool block + if workerPool.PlacementPolicyName != "" && workerPool.VGpuPolicyName != "" { + return "", fmt.Errorf("the worker pool '%s' should have either a Placement Policy or a vGPU Policy, not both", workerPool.Name) + } + placementPolicy := workerPool.PlacementPolicyName + if workerPool.VGpuPolicyName != "" { + // For convenience, we just use one of the variables as both cannot be set at same time + placementPolicy = workerPool.VGpuPolicyName + } + + if err := nodePoolEmptyTmpl.Execute(buf, map[string]string{ + "ClusterName": clusterSettings.Name, + "NodePoolName": workerPool.Name, + "TargetNamespace": clusterSettings.Name + "-ns", + "Catalog": clusterSettings.CatalogName, + "VAppTemplate": clusterSettings.KubernetesTemplateOvaName, + "NodePoolSizingPolicy": workerPool.SizingPolicyName, + "NodePoolPlacementPolicy": placementPolicy, // Can be either Placement or vGPU policy + "NodePoolStorageProfile": workerPool.StorageProfileName, + "NodePoolDiskSize": fmt.Sprintf("%dGi", workerPool.DiskSizeGi), + "NodePoolEnableGpu": strconv.FormatBool(workerPool.VGpuPolicyName != ""), + "NodePoolMachineCount": strconv.Itoa(workerPool.MachineCount), + "KubernetesVersion": clusterSettings.TkgVersionBundle.KubernetesVersion, + }); err != nil { + return "", fmt.Errorf("could not generate a correct Node Pool YAML: %s", err) + } + resultYaml += fmt.Sprintf("%s\n", buf.String()) + if i < len(clusterSettings.WorkerPools)-1 { + resultYaml += "---\n" + } + buf.Reset() + } + return resultYaml, nil +} + +// generateMachineHealthCheckYaml generates a YAML block corresponding to the cluster Machine Health Check. +// The generated YAML does not contain a separator (---) at the end. +func (clusterSettings *cseClusterSettingsInternal) generateMachineHealthCheckYaml() (string, error) { + if clusterSettings == nil { + return "", fmt.Errorf("the receiver cluster settings is nil") + } + + if clusterSettings.VcdKeConfig.NodeStartupTimeout == "" && + clusterSettings.VcdKeConfig.NodeUnknownTimeout == "" && + clusterSettings.VcdKeConfig.NodeNotReadyTimeout == "" && + clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage == 0 { + return "", nil + } + + mhcTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_mhc") + if err != nil { + return "", err + } + + mhcEmptyTmpl := template.Must(template.New(clusterSettings.Name + "-mhc").Parse(mhcTmpl)) + buf := &bytes.Buffer{} + + if err := mhcEmptyTmpl.Execute(buf, map[string]string{ + "ClusterName": clusterSettings.Name, + "TargetNamespace": clusterSettings.Name + "-ns", + // With the 'percentage' suffix + "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage), + // These values coming from VCDKEConfig (CSE Server settings) may have an "s" suffix. We make sure we don't duplicate it + "NodeStartupTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeStartupTimeout, "s", "")), + "NodeUnknownTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeUnknownTimeout, "s", "")), + "NodeNotReadyTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeNotReadyTimeout, "s", "")), + }); err != nil { + return "", fmt.Errorf("could not generate a correct Machine Health Check YAML: %s", err) + } + return fmt.Sprintf("%s\n", buf.String()), nil + +} diff --git a/govcd/cse_template_unit_test.go b/govcd/cse_internal_unit_test.go similarity index 94% rename from govcd/cse_template_unit_test.go rename to govcd/cse_internal_unit_test.go index 286e16907..6a25da2f1 100644 --- a/govcd/cse_template_unit_test.go +++ b/govcd/cse_internal_unit_test.go @@ -10,12 +10,12 @@ import ( "testing" ) +// Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { - v41, err := semver.NewVersion("4.1") + v41, err := semver.NewVersion("4.2.0") if err != nil { t.Fatalf("%s", err) } - capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") if err != nil { t.Fatalf("could not read YAML test file: %s", err) @@ -125,7 +125,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) var result []map[string]interface{} for _, doc := range baseUnmarshaledYaml { if doc["kind"] == "MachineHealthCheck" { - continue + continue // Remove the MachineHealthCheck document from the expected result } result = append(result, doc) } @@ -183,9 +183,10 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) var result []map[string]interface{} for _, doc := range baseUnmarshaledYaml { if doc["kind"] == "VCDCluster" { + // Add the extra items to the document of the expected result doc["spec"].(map[string]interface{})["controlPlaneEndpoint"] = map[string]interface{}{"host": "1.2.3.4"} doc["spec"].(map[string]interface{})["controlPlaneEndpoint"].(map[string]interface{})["port"] = 6443 - doc["spec"].(map[string]interface{})["loadBalancerConfigSpec"] = map[string]string{"vipSubnet": "6.7.8.9/24"} + doc["spec"].(map[string]interface{})["loadBalancerConfigSpec"] = map[string]interface{}{"vipSubnet": "6.7.8.9/24"} } result = append(result, doc) } @@ -227,7 +228,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) expected := tt.expectedFunc() if !reflect.DeepEqual(expected, gotUnmarshaled) { - t.Errorf("generateCapiYamlAsJsonString() got =\n%v\nwant =\n%v\n", gotUnmarshaled, expected) + t.Errorf("generateCapiYamlAsJsonString() got =\n%#v\nwant =\n%#v\n", gotUnmarshaled, expected) } }) } diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 61168ab8d..71ba5dd2c 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -220,48 +220,9 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:fd8e63dc-9127-407f-bf7b-29357442b8b4") check.Assert(err, IsNil) - workerPoolName := "test2-worker-node-pool-1" - - _, err = cluster.GetKubeconfig() - check.Assert(err, IsNil) - - // Perform the update - err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 2}}, true) - check.Assert(err, IsNil) - - // Post-check. This should be 2, as it should have scaled up - foundWorkerPool := false - for _, nodePool := range cluster.WorkerPools { - if nodePool.Name == workerPoolName { - foundWorkerPool = true - check.Assert(nodePool.MachineCount, Equals, 2) - } - } - check.Assert(foundWorkerPool, Equals, true) - - // Revert back (resources can be limited) - err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{workerPoolName: {MachineCount: 1}}, true) - check.Assert(err, IsNil) - - // Perform the update - err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 2}, true) - check.Assert(err, IsNil) - - // Post-check. This should be 2, as it should have scaled up - check.Assert(cluster.ControlPlane.MachineCount, Equals, 2) - - // Revert back (resources can be limited) - err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) + ovas, err := cluster.GetSupportedUpgrades(true) check.Assert(err, IsNil) - err = cluster.AddWorkerPools([]CseWorkerPoolSettings{{ - Name: "node-pool-5", - MachineCount: 1, - DiskSizeGi: 20, - SizingPolicyId: cluster.WorkerPools[0].SizingPolicyId, - PlacementPolicyId: "", - VGpuPolicyId: "", - StorageProfileId: cluster.WorkerPools[0].StorageProfileId, - }}, true) + err = cluster.UpgradeCluster(ovas[0].ID, true) check.Assert(err, IsNil) } diff --git a/govcd/cse_type.go b/govcd/cse_type.go index e24bd5267..db07c9cd7 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -1,7 +1,6 @@ package govcd import ( - "embed" semver "github.com/hashicorp/go-version" "github.com/vmware/go-vcloud-director/v2/types/v56" "time" @@ -26,16 +25,6 @@ type CseKubernetesCluster struct { supportedUpgrades []*types.VAppTemplate // Caches the vApp templates that can be used to upgrade a cluster. } -// CseClusterEvent is an event that has occurred during the lifetime of a Container Service Extension (CSE) Kubernetes cluster. -type CseClusterEvent struct { - Name string - Type string - ResourceId string - ResourceName string - OccurredAt time.Time - Details string -} - // CseClusterSettings defines the required configuration of a Container Service Extension (CSE) Kubernetes cluster. type CseClusterSettings struct { CseVersion semver.Version @@ -86,6 +75,16 @@ type CseDefaultStorageClassSettings struct { Filesystem string // Must be either "ext4" or "xfs" } +// CseClusterEvent is an event that has occurred during the lifetime of a Container Service Extension (CSE) Kubernetes cluster. +type CseClusterEvent struct { + Name string + Type string + ResourceId string + ResourceName string + OccurredAt time.Time + Details string +} + // CseClusterUpdateInput defines the required configuration that a Container Service Extension (CSE) Kubernetes cluster needs in order to be updated. type CseClusterUpdateInput struct { KubernetesTemplateOvaId *string @@ -110,11 +109,11 @@ type CseWorkerPoolUpdateInput struct { // cseClusterSettingsInternal defines the required arguments that are required by the CSE Server used internally to specify // a Kubernetes cluster. These are not set by the user, but instead they are computed from a valid -// CseClusterSettings object in the cseClusterSettingsToInternal method. These fields are then +// CseClusterSettings object in the CseClusterSettings.toCseClusterSettingsInternal method. These fields are then // inserted in Go templates to render a final JSON that is valid to be used as the cluster Runtime Defined Entity (RDE) payload. // // The main difference between CseClusterSettings and this structure is that the first one uses IDs and this one uses names, among -// other differences like the computed TkgVersionBundle. +// other differences like the computed tkgVersionBundle. type cseClusterSettingsInternal struct { CseVersion semver.Version Name string @@ -140,9 +139,8 @@ type cseClusterSettingsInternal struct { } // tkgVersionBundle is a type that contains all the versions of the components of -// a Kubernetes cluster that can be obtained with the Kubernetes Template OVA name, downloaded -// from VMware Customer connect: -// https://customerconnect.vmware.com/downloads/details?downloadGroup=TKG-240&productId=1400 +// a Kubernetes cluster that can be obtained with the internal properties of the Kubernetes Template OVAs downloaded from +// https://customerconnect.vmware.com type tkgVersionBundle struct { EtcdVersion string CoreDnsVersion string @@ -196,9 +194,3 @@ type cseComponentsVersions struct { CapvcdRdeTypeVersion string CseInterfaceVersion string } - -// This collection of files contains all the Go Templates and resources required for the Container Service Extension (CSE) methods -// to work. -// -//go:embed cse -var cseFiles embed.FS diff --git a/govcd/cse_util.go b/govcd/cse_util.go index a100deec2..70c514419 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -764,6 +764,8 @@ func getTkgVersionBundleFromVAppTemplate(template *types.VAppTemplate) (tkgVersi return result, nil } +// compareTkgVersion returns -1, 0 or 1 if the receiver TKG version is less than, equal or higher to the input TKG version. +// If they cannot be compared it returns -2. func (tkgVersions tkgVersionBundle) compareTkgVersion(tkgVersion string) int { receiverVersion, err := semver.NewVersion(tkgVersions.TkgVersion) if err != nil { @@ -776,6 +778,8 @@ func (tkgVersions tkgVersionBundle) compareTkgVersion(tkgVersion string) int { return receiverVersion.Compare(inputVersion) } +// kubernetesVersionIsOneMinorHigher returns true only if the receiver Kubernetes version is exactly one minor version higher +// than the given input version, being the minor digit the 'Y' in 'X.Y.Z'. func (tkgVersions tkgVersionBundle) kubernetesVersionIsOneMinorHigher(kubernetesVersion string) bool { receiverVersion, err := semver.NewVersion(tkgVersions.KubernetesVersion) if err != nil { @@ -800,7 +804,7 @@ func (tkgVersions tkgVersionBundle) kubernetesVersionIsOneMinorHigher(kubernetes // getVcdKeConfig gets the required information from the CSE Server configuration RDE (VCDKEConfig), such as the // Machine Health Check settings and the Container Registry URL. -func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheckActive bool) (*vcdKeConfig, error) { +func getVcdKeConfig(client *Client, vcdKeConfigVersion string, retrieveMachineHealtchCheckInfo bool) (*vcdKeConfig, error) { rdes, err := getRdesByName(client, "vmware", "VCDKEConfig", vcdKeConfigVersion, "vcdKeConfig") if err != nil { return nil, err @@ -822,7 +826,7 @@ func getVcdKeConfig(client *Client, vcdKeConfigVersion string, isNodeHealthCheck // https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.2/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.2/GUID-B5C19221-2ECA-4DCD-8EA1-8E391F6217C1.html result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"]) - if isNodeHealthCheckActive { + if retrieveMachineHealtchCheckInfo { mhc, ok := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"] if !ok { // If there is no "mhc" entry in the VCDKEConfig JSON, we skip setting this part of the Kubernetes cluster configuration diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 7927fd251..8b4af3cb2 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -260,7 +260,7 @@ func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernete } internalSettings.Name = cluster.Name internalSettings.CseVersion = cluster.CseVersion - nodePoolsYaml, err := internalSettings.generateNodePoolYaml() + nodePoolsYaml, err := internalSettings.generateWorkerPoolsYaml() if err != nil { return nil, err } @@ -297,7 +297,7 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, clus // We need to add the block to the slice of YAML documents settings := &cseClusterSettingsInternal{CseVersion: cseVersion, Name: clusterName, VcdKeConfig: *vcdKeConfig} - mhcYaml, err := settings.generateMemoryHealthCheckYaml() + mhcYaml, err := settings.generateMachineHealthCheckYaml() if err != nil { return nil, err } From 021a859a938100a6f427a1ae7eeb3d62758313ed Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 15 Feb 2024 16:09:48 +0100 Subject: [PATCH 053/115] Fix bug/unit test Signed-off-by: abarreiro --- govcd/cse_internal.go | 7 ++++++- govcd/cse_internal_unit_test.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/govcd/cse_internal.go b/govcd/cse_internal.go index c5e0408ee..6f5ba9f64 100644 --- a/govcd/cse_internal.go +++ b/govcd/cse_internal.go @@ -126,8 +126,13 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( if err := capiYamlEmpty.Execute(buf, args); err != nil { return "", fmt.Errorf("could not generate a correct CAPI YAML: %s", err) } + + prettyYaml := "" + if memoryHealthCheckYaml != "" { + prettyYaml += fmt.Sprintf("%s\n---\n", memoryHealthCheckYaml) + } // The final "pretty" YAML. To embed it in the final payload it must be marshaled into a one-line JSON string - prettyYaml := fmt.Sprintf("%s\n---\n%s\n---\n%s", memoryHealthCheckYaml, nodePoolYaml, buf.String()) + prettyYaml += fmt.Sprintf("%s\n---\n%s", nodePoolYaml, buf.String()) // We don't use a standard json.Marshal() as the YAML contains special characters that are not encoded properly, such as '<'. buf.Reset() diff --git a/govcd/cse_internal_unit_test.go b/govcd/cse_internal_unit_test.go index 6a25da2f1..0e0e46225 100644 --- a/govcd/cse_internal_unit_test.go +++ b/govcd/cse_internal_unit_test.go @@ -185,7 +185,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) if doc["kind"] == "VCDCluster" { // Add the extra items to the document of the expected result doc["spec"].(map[string]interface{})["controlPlaneEndpoint"] = map[string]interface{}{"host": "1.2.3.4"} - doc["spec"].(map[string]interface{})["controlPlaneEndpoint"].(map[string]interface{})["port"] = 6443 + doc["spec"].(map[string]interface{})["controlPlaneEndpoint"].(map[string]interface{})["port"] = float64(6443) doc["spec"].(map[string]interface{})["loadBalancerConfigSpec"] = map[string]interface{}{"vipSubnet": "6.7.8.9/24"} } result = append(result, doc) From 8bea6c719039fb74557dac6f598b9017ef467110 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 15 Feb 2024 16:10:14 +0100 Subject: [PATCH 054/115] Fix bug/unit test Signed-off-by: abarreiro --- govcd/cse_internal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/govcd/cse_internal.go b/govcd/cse_internal.go index 6f5ba9f64..dcd080d24 100644 --- a/govcd/cse_internal.go +++ b/govcd/cse_internal.go @@ -127,11 +127,11 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( return "", fmt.Errorf("could not generate a correct CAPI YAML: %s", err) } + // The final "pretty" YAML. To embed it in the final payload it must be marshaled into a one-line JSON string prettyYaml := "" if memoryHealthCheckYaml != "" { prettyYaml += fmt.Sprintf("%s\n---\n", memoryHealthCheckYaml) } - // The final "pretty" YAML. To embed it in the final payload it must be marshaled into a one-line JSON string prettyYaml += fmt.Sprintf("%s\n---\n%s", nodePoolYaml, buf.String()) // We don't use a standard json.Marshal() as the YAML contains special characters that are not encoded properly, such as '<'. From ad77b6e2520f0339b31d49aa9ea1ce567cb94c68 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 12:26:16 +0100 Subject: [PATCH 055/115] Fix MHC Signed-off-by: abarreiro --- govcd/cse/4.2.0/capiyaml_mhc.tmpl | 4 ++-- govcd/cse_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/govcd/cse/4.2.0/capiyaml_mhc.tmpl b/govcd/cse/4.2.0/capiyaml_mhc.tmpl index d31e4c3ec..3ff618dfb 100644 --- a/govcd/cse/4.2.0/capiyaml_mhc.tmpl +++ b/govcd/cse/4.2.0/capiyaml_mhc.tmpl @@ -1,4 +1,4 @@ -apiVersion: cluster.x-k8s.io/v1beta1 +apiVersion: cluster.x-k8s.io/v1beta2 kind: MachineHealthCheck metadata: name: "{{.ClusterName}}" @@ -19,4 +19,4 @@ spec: timeout: "{{.NodeUnknownTimeout}}" - type: Ready status: "False" - timeout: "{{.NodeNotReadyTimeout}}" \ No newline at end of file + timeout: "{{.NodeNotReadyTimeout}}" diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 71ba5dd2c..ddcaf8653 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -217,7 +217,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { } func (vcd *TestVCD) Test_Deleteme(check *C) { - cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:fd8e63dc-9127-407f-bf7b-29357442b8b4") + cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:75e996ac-fe91-49b9-8e02-73759d0c8d8a") check.Assert(err, IsNil) ovas, err := cluster.GetSupportedUpgrades(true) From b890ca6e9a7ca309942b865141b9c3cfc7bde12b Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 13:30:48 +0100 Subject: [PATCH 056/115] Fix MHC update Signed-off-by: abarreiro --- govcd/cse/4.1/capiyaml_cluster.tmpl | 25 +++++++- govcd/cse/4.1/capiyaml_mhc.tmpl | 22 ------- govcd/cse/4.2.0/capiyaml_cluster.tmpl | 25 +++++++- govcd/cse/4.2.0/capiyaml_mhc.tmpl | 22 ------- govcd/cse_internal.go | 44 +++++--------- govcd/cse_internal_unit_test.go | 11 +++- govcd/cse_test.go | 2 + govcd/cse_type.go | 3 +- govcd/cse_util.go | 23 ++++---- govcd/cse_yaml.go | 62 +++++++------------- govcd/cse_yaml_unit_test.go | 84 +++++++++++++++++---------- 11 files changed, 163 insertions(+), 160 deletions(-) delete mode 100644 govcd/cse/4.1/capiyaml_mhc.tmpl delete mode 100644 govcd/cse/4.2.0/capiyaml_mhc.tmpl diff --git a/govcd/cse/4.1/capiyaml_cluster.tmpl b/govcd/cse/4.1/capiyaml_cluster.tmpl index 16a676ae1..9243da039 100644 --- a/govcd/cse/4.1/capiyaml_cluster.tmpl +++ b/govcd/cse/4.1/capiyaml_cluster.tmpl @@ -1,4 +1,27 @@ apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" + labels: + clusterctl.cluster.x-k8s.io: "" + clusterctl.cluster.x-k8s.io/move: "" +spec: + clusterName: "{{.ClusterName}}" + maxUnhealthy: "{{.MaxUnhealthyNodePercentage}}" + nodeStartupTimeout: "{{.NodeStartupTimeout}}" + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: "{{.ClusterName}}" + unhealthyConditions: + - type: Ready + status: Unknown + timeout: "{{.NodeUnknownTimeout}}" + - type: Ready + status: "False" + timeout: "{{.NodeNotReadyTimeout}}" +--- +apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: "{{.ClusterName}}" @@ -150,4 +173,4 @@ spec: criSocket: /run/containerd/containerd.sock kubeletExtraArgs: eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% - cloud-provider: external \ No newline at end of file + cloud-provider: external diff --git a/govcd/cse/4.1/capiyaml_mhc.tmpl b/govcd/cse/4.1/capiyaml_mhc.tmpl deleted file mode 100644 index d31e4c3ec..000000000 --- a/govcd/cse/4.1/capiyaml_mhc.tmpl +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: "{{.ClusterName}}" - namespace: "{{.TargetNamespace}}" - labels: - clusterctl.cluster.x-k8s.io: "" - clusterctl.cluster.x-k8s.io/move: "" -spec: - clusterName: "{{.ClusterName}}" - maxUnhealthy: "{{.MaxUnhealthyNodePercentage}}" - nodeStartupTimeout: "{{.NodeStartupTimeout}}" - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: "{{.ClusterName}}" - unhealthyConditions: - - type: Ready - status: Unknown - timeout: "{{.NodeUnknownTimeout}}" - - type: Ready - status: "False" - timeout: "{{.NodeNotReadyTimeout}}" \ No newline at end of file diff --git a/govcd/cse/4.2.0/capiyaml_cluster.tmpl b/govcd/cse/4.2.0/capiyaml_cluster.tmpl index 16a676ae1..d1df931c3 100644 --- a/govcd/cse/4.2.0/capiyaml_cluster.tmpl +++ b/govcd/cse/4.2.0/capiyaml_cluster.tmpl @@ -1,3 +1,26 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: MachineHealthCheck +metadata: + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" + labels: + clusterctl.cluster.x-k8s.io: "" + clusterctl.cluster.x-k8s.io/move: "" +spec: + clusterName: "{{.ClusterName}}" + maxUnhealthy: "{{.MaxUnhealthyNodePercentage}}" + nodeStartupTimeout: "{{.NodeStartupTimeout}}" + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: "{{.ClusterName}}" + unhealthyConditions: + - type: Ready + status: Unknown + timeout: "{{.NodeUnknownTimeout}}" + - type: Ready + status: "False" + timeout: "{{.NodeNotReadyTimeout}}" +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: @@ -150,4 +173,4 @@ spec: criSocket: /run/containerd/containerd.sock kubeletExtraArgs: eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% - cloud-provider: external \ No newline at end of file + cloud-provider: external diff --git a/govcd/cse/4.2.0/capiyaml_mhc.tmpl b/govcd/cse/4.2.0/capiyaml_mhc.tmpl deleted file mode 100644 index 3ff618dfb..000000000 --- a/govcd/cse/4.2.0/capiyaml_mhc.tmpl +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta2 -kind: MachineHealthCheck -metadata: - name: "{{.ClusterName}}" - namespace: "{{.TargetNamespace}}" - labels: - clusterctl.cluster.x-k8s.io: "" - clusterctl.cluster.x-k8s.io/move: "" -spec: - clusterName: "{{.ClusterName}}" - maxUnhealthy: "{{.MaxUnhealthyNodePercentage}}" - nodeStartupTimeout: "{{.NodeStartupTimeout}}" - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: "{{.ClusterName}}" - unhealthyConditions: - - type: Ready - status: Unknown - timeout: "{{.NodeUnknownTimeout}}" - - type: Ready - status: "False" - timeout: "{{.NodeNotReadyTimeout}}" diff --git a/govcd/cse_internal.go b/govcd/cse_internal.go index dcd080d24..8cde4bacb 100644 --- a/govcd/cse_internal.go +++ b/govcd/cse_internal.go @@ -88,7 +88,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( return "", err } - memoryHealthCheckYaml, err := clusterSettings.generateMachineHealthCheckYaml() + memoryHealthCheckParameters, err := clusterSettings.getMachineHealthTemplateParameters() if err != nil { return "", err } @@ -121,6 +121,9 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( "SshPublicKey": clusterSettings.SshPublicKey, "VirtualIpSubnet": clusterSettings.VirtualIpSubnet, } + for k, v := range memoryHealthCheckParameters { + args[k] = v + } buf := &bytes.Buffer{} if err := capiYamlEmpty.Execute(buf, args); err != nil { @@ -128,11 +131,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( } // The final "pretty" YAML. To embed it in the final payload it must be marshaled into a one-line JSON string - prettyYaml := "" - if memoryHealthCheckYaml != "" { - prettyYaml += fmt.Sprintf("%s\n---\n", memoryHealthCheckYaml) - } - prettyYaml += fmt.Sprintf("%s\n---\n%s", nodePoolYaml, buf.String()) + prettyYaml := fmt.Sprintf("%s\n---\n%s", nodePoolYaml, buf.String()) // We don't use a standard json.Marshal() as the YAML contains special characters that are not encoded properly, such as '<'. buf.Reset() @@ -201,40 +200,29 @@ func (clusterSettings *cseClusterSettingsInternal) generateWorkerPoolsYaml() (st return resultYaml, nil } -// generateMachineHealthCheckYaml generates a YAML block corresponding to the cluster Machine Health Check. -// The generated YAML does not contain a separator (---) at the end. -func (clusterSettings *cseClusterSettingsInternal) generateMachineHealthCheckYaml() (string, error) { +// getMachineHealthTemplateParameters generates the required parameters for the YAML block corresponding to the cluster Machine Health Check. +func (clusterSettings *cseClusterSettingsInternal) getMachineHealthTemplateParameters() (map[string]string, error) { if clusterSettings == nil { - return "", fmt.Errorf("the receiver cluster settings is nil") - } - - if clusterSettings.VcdKeConfig.NodeStartupTimeout == "" && - clusterSettings.VcdKeConfig.NodeUnknownTimeout == "" && - clusterSettings.VcdKeConfig.NodeNotReadyTimeout == "" && - clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage == 0 { - return "", nil + return nil, fmt.Errorf("the receiver cluster settings is nil") } - mhcTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_mhc") - if err != nil { - return "", err + // If the Machine Health Check is deactivated, it is enough to set 'spec.maxUnhealthy' to '0%' in the YAML + // to deactivate health checks. + maxUnhealthy := clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage + if !clusterSettings.MachineHealthCheckEnabled { + maxUnhealthy = 0 } - mhcEmptyTmpl := template.Must(template.New(clusterSettings.Name + "-mhc").Parse(mhcTmpl)) - buf := &bytes.Buffer{} - - if err := mhcEmptyTmpl.Execute(buf, map[string]string{ + mhcSettings := map[string]string{ "ClusterName": clusterSettings.Name, "TargetNamespace": clusterSettings.Name + "-ns", // With the 'percentage' suffix - "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage), + "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", maxUnhealthy), // These values coming from VCDKEConfig (CSE Server settings) may have an "s" suffix. We make sure we don't duplicate it "NodeStartupTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeStartupTimeout, "s", "")), "NodeUnknownTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeUnknownTimeout, "s", "")), "NodeNotReadyTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeNotReadyTimeout, "s", "")), - }); err != nil { - return "", fmt.Errorf("could not generate a correct Machine Health Check YAML: %s", err) } - return fmt.Sprintf("%s\n", buf.String()), nil + return mhcSettings, nil } diff --git a/govcd/cse_internal_unit_test.go b/govcd/cse_internal_unit_test.go index 0e0e46225..87cffbca3 100644 --- a/govcd/cse_internal_unit_test.go +++ b/govcd/cse_internal_unit_test.go @@ -63,6 +63,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) StorageProfileName: "*", }, }, + MachineHealthCheckEnabled: true, VcdKeConfig: vcdKeConfig{ MaxUnhealthyNodesPercentage: 100, NodeStartupTimeout: "900", @@ -112,8 +113,13 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) StorageProfileName: "*", }, }, + MachineHealthCheckEnabled: false, VcdKeConfig: vcdKeConfig{ - ContainerRegistryUrl: "projects.registry.vmware.com/tkg", + MaxUnhealthyNodesPercentage: 66, + NodeStartupTimeout: "100s", + NodeNotReadyTimeout: "200s", + NodeUnknownTimeout: "300s", + ContainerRegistryUrl: "projects.registry.vmware.com/tkg", }, Owner: "dummy", ApiToken: "dummy", @@ -125,7 +131,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) var result []map[string]interface{} for _, doc := range baseUnmarshaledYaml { if doc["kind"] == "MachineHealthCheck" { - continue // Remove the MachineHealthCheck document from the expected result + doc["spec"].(map[string]interface{})["maxUnhealthy"] = "0%" } result = append(result, doc) } @@ -165,6 +171,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) StorageProfileName: "*", }, }, + MachineHealthCheckEnabled: true, VcdKeConfig: vcdKeConfig{ MaxUnhealthyNodesPercentage: 100, NodeStartupTimeout: "900", diff --git a/govcd/cse_test.go b/govcd/cse_test.go index ddcaf8653..ae18e626f 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -225,4 +225,6 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { err = cluster.UpgradeCluster(ovas[0].ID, true) check.Assert(err, IsNil) + + cluster.SetHealthCheck(false, true) } diff --git a/govcd/cse_type.go b/govcd/cse_type.go index db07c9cd7..2054d9890 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -127,7 +127,6 @@ type cseClusterSettingsInternal struct { ControlPlane cseControlPlaneSettingsInternal WorkerPools []cseWorkerPoolSettingsInternal DefaultStorageClass cseDefaultStorageClassInternal - VcdKeConfig vcdKeConfig Owner string ApiToken string VcdUrl string @@ -135,7 +134,9 @@ type cseClusterSettingsInternal struct { SshPublicKey string PodCidr string ServiceCidr string + MachineHealthCheckEnabled bool AutoRepairOnErrors bool + VcdKeConfig vcdKeConfig } // tkgVersionBundle is a type that contains all the versions of the components of diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 70c514419..b47553433 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -682,7 +682,8 @@ func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClus } } - vcdKeConfig, err := getVcdKeConfig(org.client, cseComponentsVersions.VcdKeConfigRdeTypeVersion, input.NodeHealthCheck) + output.MachineHealthCheckEnabled = input.NodeHealthCheck + vcdKeConfig, err := getVcdKeConfig(org.client, cseComponentsVersions.VcdKeConfigRdeTypeVersion) if err != nil { return nil, err } @@ -804,7 +805,7 @@ func (tkgVersions tkgVersionBundle) kubernetesVersionIsOneMinorHigher(kubernetes // getVcdKeConfig gets the required information from the CSE Server configuration RDE (VCDKEConfig), such as the // Machine Health Check settings and the Container Registry URL. -func getVcdKeConfig(client *Client, vcdKeConfigVersion string, retrieveMachineHealtchCheckInfo bool) (*vcdKeConfig, error) { +func getVcdKeConfig(client *Client, vcdKeConfigVersion string) (*vcdKeConfig, error) { rdes, err := getRdesByName(client, "vmware", "VCDKEConfig", vcdKeConfigVersion, "vcdKeConfig") if err != nil { return nil, err @@ -826,17 +827,15 @@ func getVcdKeConfig(client *Client, vcdKeConfigVersion string, retrieveMachineHe // https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.2/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.2/GUID-B5C19221-2ECA-4DCD-8EA1-8E391F6217C1.html result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"]) - if retrieveMachineHealtchCheckInfo { - mhc, ok := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"] - if !ok { - // If there is no "mhc" entry in the VCDKEConfig JSON, we skip setting this part of the Kubernetes cluster configuration - return result, nil - } - result.MaxUnhealthyNodesPercentage = mhc.(map[string]interface{})["maxUnhealthyNodes"].(float64) - result.NodeStartupTimeout = mhc.(map[string]interface{})["nodeStartupTimeout"].(string) - result.NodeNotReadyTimeout = mhc.(map[string]interface{})["nodeUnknownTimeout"].(string) - result.NodeUnknownTimeout = mhc.(map[string]interface{})["nodeNotReadyTimeout"].(string) + mhc, ok := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"] + if !ok { + // If there is no "mhc" entry in the VCDKEConfig JSON, we skip setting this part of the Kubernetes cluster configuration + return result, nil } + result.MaxUnhealthyNodesPercentage = mhc.(map[string]interface{})["maxUnhealthyNodes"].(float64) + result.NodeStartupTimeout = mhc.(map[string]interface{})["nodeStartupTimeout"].(string) + result.NodeNotReadyTimeout = mhc.(map[string]interface{})["nodeUnknownTimeout"].(string) + result.NodeUnknownTimeout = mhc.(map[string]interface{})["nodeNotReadyTimeout"].(string) return result, nil } diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 8b4af3cb2..29a421d51 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -2,7 +2,6 @@ package govcd import ( "fmt" - semver "github.com/hashicorp/go-version" "github.com/vmware/go-vcloud-director/v2/types/v56" "sigs.k8s.io/yaml" "strings" @@ -83,11 +82,11 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) } if input.NodeHealthCheck != nil { - vcdKeConfig, err := getVcdKeConfig(cluster.client, cluster.capvcdType.Status.VcdKe.VcdKeVersion, *input.NodeHealthCheck) + vcdKeConfig, err := getVcdKeConfig(cluster.client, cluster.capvcdType.Status.VcdKe.VcdKeVersion) if err != nil { return "", err } - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, cluster.Name, cluster.CseVersion, vcdKeConfig) + err = cseUpdateNodeHealthCheckInYaml(yamlDocs, vcdKeConfig, cluster.NodeHealthCheck) if err != nil { return "", err } @@ -278,49 +277,32 @@ func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernete // cseUpdateNodeHealthCheckInYaml updates the Kubernetes cluster described in the given YAML documents by adding or removing // the MachineHealthCheck object. // NOTE: This function doesn't modify the input, but returns a copy of the YAML with the modifications. -func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, clusterName string, cseVersion semver.Version, vcdKeConfig *vcdKeConfig) ([]map[string]interface{}, error) { - mhcPosition := -1 - result := make([]map[string]interface{}, len(yamlDocuments)) - for i, d := range yamlDocuments { - if d["kind"] == "MachineHealthCheck" { - mhcPosition = i +func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, vcdKeConfig *vcdKeConfig, mhcEnabled bool) error { + for _, d := range yamlDocuments { + if d["kind"] != "MachineHealthCheck" { + continue } - result[i] = d - } - if mhcPosition < 0 { - // There is no MachineHealthCheck block - if vcdKeConfig == nil { - // We don't want it neither, so nothing to do - return result, nil + maxUnhealthy := vcdKeConfig.MaxUnhealthyNodesPercentage + if !mhcEnabled { + maxUnhealthy = 0 } - // We need to add the block to the slice of YAML documents - settings := &cseClusterSettingsInternal{CseVersion: cseVersion, Name: clusterName, VcdKeConfig: *vcdKeConfig} - mhcYaml, err := settings.generateMachineHealthCheckYaml() - if err != nil { - return nil, err - } - var mhc map[string]interface{} - err = yaml.Unmarshal([]byte(mhcYaml), &mhc) - if err != nil { - return nil, err - } - result = append(result, mhc) - } else { - // There is a MachineHealthCheck block - if vcdKeConfig != nil { - // We want it, but it is already there, so nothing to do - // TODO: What happens in UI if the VCDKEConfig MHC values are changed, does it get reflected in the cluster? - // If that's the case, we might need to update this value always - return result, nil + // Replace them in the YAML + d["spec"].(map[string]interface{})["maxUnhealthy"] = fmt.Sprintf("%.0f%%", maxUnhealthy) + d["spec"].(map[string]interface{})["nodeStartupTimeout"] = fmt.Sprintf("%ss", strings.ReplaceAll(vcdKeConfig.NodeStartupTimeout, "s", "")) + unhealthyConditions := traverseMapAndGet[[]interface{}](d, "spec.unhealthyConditions") + for _, uc := range unhealthyConditions { + ucBlock := uc.(map[string]interface{}) + if ucBlock["status"] == "Unknown" { + ucBlock["timeout"] = fmt.Sprintf("%ss", strings.ReplaceAll(vcdKeConfig.NodeUnknownTimeout, "s", "")) + } + if ucBlock["status"] == "\"Ready\"" { + ucBlock["timeout"] = fmt.Sprintf("%ss", strings.ReplaceAll(vcdKeConfig.NodeNotReadyTimeout, "s", "")) + } } - - // We don't want Machine Health Checks, we delete the YAML document - result[mhcPosition] = result[len(result)-1] // We override the MachineHealthCheck block with the last document - result = result[:len(result)-1] // We remove the last document (now duplicated) } - return result, nil + return nil } // marshalMultipleYamlDocuments takes a slice of maps representing multiple YAML documents (one per item in the slice) and diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 677145960..7ff0a738b 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -3,7 +3,6 @@ package govcd import ( - semver "github.com/hashicorp/go-version" "github.com/vmware/go-vcloud-director/v2/types/v56" "os" "reflect" @@ -263,53 +262,76 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { t.Fatal("could not find the cluster name in the CAPI YAML test file") } - v, err := semver.NewVersion("4.1") - if err != nil { - t.Fatalf("incorrect version: %s", err) - } - // Deactivates Machine Health Check - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, nil) + err = cseUpdateNodeHealthCheckInYaml(yamlDocs, &vcdKeConfig{ + MaxUnhealthyNodesPercentage: 66, + NodeStartupTimeout: "100s", + NodeNotReadyTimeout: "200s", + NodeUnknownTimeout: "300s", + }, false) if err != nil { t.Fatalf("%s", err) } - // The resulting documents should not have that document for _, document := range yamlDocs { if document["kind"] == "MachineHealthCheck" { - t.Fatal("Expected the MachineHealthCheck to be deleted, but it is there") + if traverseMapAndGet[string](document, "spec.maxUnhealthy") != "0%" { + t.Fatalf("expected spec.maxUnhealthy to be updated to 0%%") + } + if traverseMapAndGet[string](document, "spec.nodeStartupTimeout") != "100s" { + t.Fatalf("expected spec.nodeStartupTimeout to remain at 100s") + } + unhealthyConditions := traverseMapAndGet[[]interface{}](document, "spec.unhealthyConditions") + for i, uc := range unhealthyConditions { + ucBlock := uc.(map[string]interface{}) + if ucBlock["status"] == "Unknown" { + if ucBlock["timeout"] != "300s" { + t.Fatalf("expected spec.unhealthyConditions[%d].timeout of status unknown to remain at 300s", i) + } + } + if ucBlock["status"] == "\"Ready\"" { + if ucBlock["timeout"] != "200s" { + t.Fatalf("expected spec.unhealthyConditions[%d].timeout to status ready to remain at 200s", i) + } + } + } } } // Enables Machine Health Check - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, &vcdKeConfig{ - MaxUnhealthyNodesPercentage: 12, - NodeStartupTimeout: "34", - NodeNotReadyTimeout: "56", - NodeUnknownTimeout: "78", - }) + err = cseUpdateNodeHealthCheckInYaml(yamlDocs, &vcdKeConfig{ + MaxUnhealthyNodesPercentage: 66, + NodeStartupTimeout: "100s", + NodeNotReadyTimeout: "200s", + NodeUnknownTimeout: "300s", + }, true) if err != nil { t.Fatalf("%s", err) } - // The resulting documents should have a MachineHealthCheck - found := false for _, document := range yamlDocs { - if document["kind"] != "MachineHealthCheck" { - continue - } - maxUnhealthy := traverseMapAndGet[string](document, "spec.maxUnhealthy") - if maxUnhealthy != "12%" { - t.Fatalf("expected a 'spec.maxUnhealthy' = 12%%, but got %s", maxUnhealthy) - } - nodeStartupTimeout := traverseMapAndGet[string](document, "spec.nodeStartupTimeout") - if nodeStartupTimeout != "34s" { - t.Fatalf("expected a 'spec.nodeStartupTimeout' = 34s, but got %s", nodeStartupTimeout) + if document["kind"] == "MachineHealthCheck" { + if traverseMapAndGet[string](document, "spec.maxUnhealthy") != "66%" { + t.Fatalf("expected spec.maxUnhealthy to be updated to 66%%") + } + if traverseMapAndGet[string](document, "spec.nodeStartupTimeout") != "100s" { + t.Fatalf("expected spec.nodeStartupTimeout to remain at 100s") + } + unhealthyConditions := traverseMapAndGet[[]interface{}](document, "spec.unhealthyConditions") + for i, uc := range unhealthyConditions { + ucBlock := uc.(map[string]interface{}) + if ucBlock["status"] == "Unknown" { + if ucBlock["timeout"] != "300s" { + t.Fatalf("expected spec.unhealthyConditions[%d].timeout of status unknown to remain at 300s", i) + } + } + if ucBlock["status"] == "\"Ready\"" { + if ucBlock["timeout"] != "200s" { + t.Fatalf("expected spec.unhealthyConditions[%d].timeout to status ready to remain at 200s", i) + } + } + } } - found = true - } - if !found { - t.Fatalf("expected a MachineHealthCheck block but got nothing") } } From 08dd63117ffc977af96c7ebacbef62a2d3482ae7 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 13:43:04 +0100 Subject: [PATCH 057/115] Fix MHC templates Signed-off-by: abarreiro --- govcd/cse/4.1/capiyaml_cluster.tmpl | 2 +- govcd/cse/4.2.0/capiyaml_cluster.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/govcd/cse/4.1/capiyaml_cluster.tmpl b/govcd/cse/4.1/capiyaml_cluster.tmpl index 9243da039..c024748b8 100644 --- a/govcd/cse/4.1/capiyaml_cluster.tmpl +++ b/govcd/cse/4.1/capiyaml_cluster.tmpl @@ -1,7 +1,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: - name: "{{.ClusterName}}" + name: "{{.ClusterName}}-mhc" namespace: "{{.TargetNamespace}}" labels: clusterctl.cluster.x-k8s.io: "" diff --git a/govcd/cse/4.2.0/capiyaml_cluster.tmpl b/govcd/cse/4.2.0/capiyaml_cluster.tmpl index d1df931c3..336351d1b 100644 --- a/govcd/cse/4.2.0/capiyaml_cluster.tmpl +++ b/govcd/cse/4.2.0/capiyaml_cluster.tmpl @@ -1,7 +1,7 @@ apiVersion: cluster.x-k8s.io/v1beta2 kind: MachineHealthCheck metadata: - name: "{{.ClusterName}}" + name: "{{.ClusterName}}-mhc" namespace: "{{.TargetNamespace}}" labels: clusterctl.cluster.x-k8s.io: "" From 0b917b3794dd6dee5aee221a314561bb5d8a37bf Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 15:15:57 +0100 Subject: [PATCH 058/115] Revert Signed-off-by: abarreiro --- govcd/cse/4.1/capiyaml_cluster.tmpl | 2 +- govcd/cse/4.2.0/capiyaml_cluster.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/govcd/cse/4.1/capiyaml_cluster.tmpl b/govcd/cse/4.1/capiyaml_cluster.tmpl index c024748b8..9243da039 100644 --- a/govcd/cse/4.1/capiyaml_cluster.tmpl +++ b/govcd/cse/4.1/capiyaml_cluster.tmpl @@ -1,7 +1,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: - name: "{{.ClusterName}}-mhc" + name: "{{.ClusterName}}" namespace: "{{.TargetNamespace}}" labels: clusterctl.cluster.x-k8s.io: "" diff --git a/govcd/cse/4.2.0/capiyaml_cluster.tmpl b/govcd/cse/4.2.0/capiyaml_cluster.tmpl index 336351d1b..d1df931c3 100644 --- a/govcd/cse/4.2.0/capiyaml_cluster.tmpl +++ b/govcd/cse/4.2.0/capiyaml_cluster.tmpl @@ -1,7 +1,7 @@ apiVersion: cluster.x-k8s.io/v1beta2 kind: MachineHealthCheck metadata: - name: "{{.ClusterName}}-mhc" + name: "{{.ClusterName}}" namespace: "{{.TargetNamespace}}" labels: clusterctl.cluster.x-k8s.io: "" From d6dbb6fd8f804e837108736f84de67dde1272310 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 15:16:12 +0100 Subject: [PATCH 059/115] revert Signed-off-by: abarreiro --- govcd/cse/4.1/capiyaml_cluster.tmpl | 25 +------- govcd/cse/4.1/capiyaml_mhc.tmpl | 22 +++++++ govcd/cse/4.2.0/capiyaml_cluster.tmpl | 25 +------- govcd/cse/4.2.0/capiyaml_mhc.tmpl | 22 +++++++ govcd/cse_internal.go | 44 +++++++++----- govcd/cse_internal_unit_test.go | 11 +--- govcd/cse_test.go | 2 - govcd/cse_type.go | 3 +- govcd/cse_util.go | 23 ++++---- govcd/cse_yaml.go | 62 +++++++++++++------- govcd/cse_yaml_unit_test.go | 84 ++++++++++----------------- 11 files changed, 160 insertions(+), 163 deletions(-) create mode 100644 govcd/cse/4.1/capiyaml_mhc.tmpl create mode 100644 govcd/cse/4.2.0/capiyaml_mhc.tmpl diff --git a/govcd/cse/4.1/capiyaml_cluster.tmpl b/govcd/cse/4.1/capiyaml_cluster.tmpl index 9243da039..16a676ae1 100644 --- a/govcd/cse/4.1/capiyaml_cluster.tmpl +++ b/govcd/cse/4.1/capiyaml_cluster.tmpl @@ -1,27 +1,4 @@ apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: "{{.ClusterName}}" - namespace: "{{.TargetNamespace}}" - labels: - clusterctl.cluster.x-k8s.io: "" - clusterctl.cluster.x-k8s.io/move: "" -spec: - clusterName: "{{.ClusterName}}" - maxUnhealthy: "{{.MaxUnhealthyNodePercentage}}" - nodeStartupTimeout: "{{.NodeStartupTimeout}}" - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: "{{.ClusterName}}" - unhealthyConditions: - - type: Ready - status: Unknown - timeout: "{{.NodeUnknownTimeout}}" - - type: Ready - status: "False" - timeout: "{{.NodeNotReadyTimeout}}" ---- -apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: "{{.ClusterName}}" @@ -173,4 +150,4 @@ spec: criSocket: /run/containerd/containerd.sock kubeletExtraArgs: eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% - cloud-provider: external + cloud-provider: external \ No newline at end of file diff --git a/govcd/cse/4.1/capiyaml_mhc.tmpl b/govcd/cse/4.1/capiyaml_mhc.tmpl new file mode 100644 index 000000000..d31e4c3ec --- /dev/null +++ b/govcd/cse/4.1/capiyaml_mhc.tmpl @@ -0,0 +1,22 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" + labels: + clusterctl.cluster.x-k8s.io: "" + clusterctl.cluster.x-k8s.io/move: "" +spec: + clusterName: "{{.ClusterName}}" + maxUnhealthy: "{{.MaxUnhealthyNodePercentage}}" + nodeStartupTimeout: "{{.NodeStartupTimeout}}" + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: "{{.ClusterName}}" + unhealthyConditions: + - type: Ready + status: Unknown + timeout: "{{.NodeUnknownTimeout}}" + - type: Ready + status: "False" + timeout: "{{.NodeNotReadyTimeout}}" \ No newline at end of file diff --git a/govcd/cse/4.2.0/capiyaml_cluster.tmpl b/govcd/cse/4.2.0/capiyaml_cluster.tmpl index d1df931c3..16a676ae1 100644 --- a/govcd/cse/4.2.0/capiyaml_cluster.tmpl +++ b/govcd/cse/4.2.0/capiyaml_cluster.tmpl @@ -1,26 +1,3 @@ -apiVersion: cluster.x-k8s.io/v1beta2 -kind: MachineHealthCheck -metadata: - name: "{{.ClusterName}}" - namespace: "{{.TargetNamespace}}" - labels: - clusterctl.cluster.x-k8s.io: "" - clusterctl.cluster.x-k8s.io/move: "" -spec: - clusterName: "{{.ClusterName}}" - maxUnhealthy: "{{.MaxUnhealthyNodePercentage}}" - nodeStartupTimeout: "{{.NodeStartupTimeout}}" - selector: - matchLabels: - cluster.x-k8s.io/cluster-name: "{{.ClusterName}}" - unhealthyConditions: - - type: Ready - status: Unknown - timeout: "{{.NodeUnknownTimeout}}" - - type: Ready - status: "False" - timeout: "{{.NodeNotReadyTimeout}}" ---- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: @@ -173,4 +150,4 @@ spec: criSocket: /run/containerd/containerd.sock kubeletExtraArgs: eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% - cloud-provider: external + cloud-provider: external \ No newline at end of file diff --git a/govcd/cse/4.2.0/capiyaml_mhc.tmpl b/govcd/cse/4.2.0/capiyaml_mhc.tmpl new file mode 100644 index 000000000..3ff618dfb --- /dev/null +++ b/govcd/cse/4.2.0/capiyaml_mhc.tmpl @@ -0,0 +1,22 @@ +apiVersion: cluster.x-k8s.io/v1beta2 +kind: MachineHealthCheck +metadata: + name: "{{.ClusterName}}" + namespace: "{{.TargetNamespace}}" + labels: + clusterctl.cluster.x-k8s.io: "" + clusterctl.cluster.x-k8s.io/move: "" +spec: + clusterName: "{{.ClusterName}}" + maxUnhealthy: "{{.MaxUnhealthyNodePercentage}}" + nodeStartupTimeout: "{{.NodeStartupTimeout}}" + selector: + matchLabels: + cluster.x-k8s.io/cluster-name: "{{.ClusterName}}" + unhealthyConditions: + - type: Ready + status: Unknown + timeout: "{{.NodeUnknownTimeout}}" + - type: Ready + status: "False" + timeout: "{{.NodeNotReadyTimeout}}" diff --git a/govcd/cse_internal.go b/govcd/cse_internal.go index 8cde4bacb..dcd080d24 100644 --- a/govcd/cse_internal.go +++ b/govcd/cse_internal.go @@ -88,7 +88,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( return "", err } - memoryHealthCheckParameters, err := clusterSettings.getMachineHealthTemplateParameters() + memoryHealthCheckYaml, err := clusterSettings.generateMachineHealthCheckYaml() if err != nil { return "", err } @@ -121,9 +121,6 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( "SshPublicKey": clusterSettings.SshPublicKey, "VirtualIpSubnet": clusterSettings.VirtualIpSubnet, } - for k, v := range memoryHealthCheckParameters { - args[k] = v - } buf := &bytes.Buffer{} if err := capiYamlEmpty.Execute(buf, args); err != nil { @@ -131,7 +128,11 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( } // The final "pretty" YAML. To embed it in the final payload it must be marshaled into a one-line JSON string - prettyYaml := fmt.Sprintf("%s\n---\n%s", nodePoolYaml, buf.String()) + prettyYaml := "" + if memoryHealthCheckYaml != "" { + prettyYaml += fmt.Sprintf("%s\n---\n", memoryHealthCheckYaml) + } + prettyYaml += fmt.Sprintf("%s\n---\n%s", nodePoolYaml, buf.String()) // We don't use a standard json.Marshal() as the YAML contains special characters that are not encoded properly, such as '<'. buf.Reset() @@ -200,29 +201,40 @@ func (clusterSettings *cseClusterSettingsInternal) generateWorkerPoolsYaml() (st return resultYaml, nil } -// getMachineHealthTemplateParameters generates the required parameters for the YAML block corresponding to the cluster Machine Health Check. -func (clusterSettings *cseClusterSettingsInternal) getMachineHealthTemplateParameters() (map[string]string, error) { +// generateMachineHealthCheckYaml generates a YAML block corresponding to the cluster Machine Health Check. +// The generated YAML does not contain a separator (---) at the end. +func (clusterSettings *cseClusterSettingsInternal) generateMachineHealthCheckYaml() (string, error) { if clusterSettings == nil { - return nil, fmt.Errorf("the receiver cluster settings is nil") + return "", fmt.Errorf("the receiver cluster settings is nil") + } + + if clusterSettings.VcdKeConfig.NodeStartupTimeout == "" && + clusterSettings.VcdKeConfig.NodeUnknownTimeout == "" && + clusterSettings.VcdKeConfig.NodeNotReadyTimeout == "" && + clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage == 0 { + return "", nil } - // If the Machine Health Check is deactivated, it is enough to set 'spec.maxUnhealthy' to '0%' in the YAML - // to deactivate health checks. - maxUnhealthy := clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage - if !clusterSettings.MachineHealthCheckEnabled { - maxUnhealthy = 0 + mhcTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_mhc") + if err != nil { + return "", err } - mhcSettings := map[string]string{ + mhcEmptyTmpl := template.Must(template.New(clusterSettings.Name + "-mhc").Parse(mhcTmpl)) + buf := &bytes.Buffer{} + + if err := mhcEmptyTmpl.Execute(buf, map[string]string{ "ClusterName": clusterSettings.Name, "TargetNamespace": clusterSettings.Name + "-ns", // With the 'percentage' suffix - "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", maxUnhealthy), + "MaxUnhealthyNodePercentage": fmt.Sprintf("%.0f%%", clusterSettings.VcdKeConfig.MaxUnhealthyNodesPercentage), // These values coming from VCDKEConfig (CSE Server settings) may have an "s" suffix. We make sure we don't duplicate it "NodeStartupTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeStartupTimeout, "s", "")), "NodeUnknownTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeUnknownTimeout, "s", "")), "NodeNotReadyTimeout": fmt.Sprintf("%ss", strings.ReplaceAll(clusterSettings.VcdKeConfig.NodeNotReadyTimeout, "s", "")), + }); err != nil { + return "", fmt.Errorf("could not generate a correct Machine Health Check YAML: %s", err) } - return mhcSettings, nil + return fmt.Sprintf("%s\n", buf.String()), nil } diff --git a/govcd/cse_internal_unit_test.go b/govcd/cse_internal_unit_test.go index 87cffbca3..0e0e46225 100644 --- a/govcd/cse_internal_unit_test.go +++ b/govcd/cse_internal_unit_test.go @@ -63,7 +63,6 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) StorageProfileName: "*", }, }, - MachineHealthCheckEnabled: true, VcdKeConfig: vcdKeConfig{ MaxUnhealthyNodesPercentage: 100, NodeStartupTimeout: "900", @@ -113,13 +112,8 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) StorageProfileName: "*", }, }, - MachineHealthCheckEnabled: false, VcdKeConfig: vcdKeConfig{ - MaxUnhealthyNodesPercentage: 66, - NodeStartupTimeout: "100s", - NodeNotReadyTimeout: "200s", - NodeUnknownTimeout: "300s", - ContainerRegistryUrl: "projects.registry.vmware.com/tkg", + ContainerRegistryUrl: "projects.registry.vmware.com/tkg", }, Owner: "dummy", ApiToken: "dummy", @@ -131,7 +125,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) var result []map[string]interface{} for _, doc := range baseUnmarshaledYaml { if doc["kind"] == "MachineHealthCheck" { - doc["spec"].(map[string]interface{})["maxUnhealthy"] = "0%" + continue // Remove the MachineHealthCheck document from the expected result } result = append(result, doc) } @@ -171,7 +165,6 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) StorageProfileName: "*", }, }, - MachineHealthCheckEnabled: true, VcdKeConfig: vcdKeConfig{ MaxUnhealthyNodesPercentage: 100, NodeStartupTimeout: "900", diff --git a/govcd/cse_test.go b/govcd/cse_test.go index ae18e626f..ddcaf8653 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -225,6 +225,4 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { err = cluster.UpgradeCluster(ovas[0].ID, true) check.Assert(err, IsNil) - - cluster.SetHealthCheck(false, true) } diff --git a/govcd/cse_type.go b/govcd/cse_type.go index 2054d9890..db07c9cd7 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -127,6 +127,7 @@ type cseClusterSettingsInternal struct { ControlPlane cseControlPlaneSettingsInternal WorkerPools []cseWorkerPoolSettingsInternal DefaultStorageClass cseDefaultStorageClassInternal + VcdKeConfig vcdKeConfig Owner string ApiToken string VcdUrl string @@ -134,9 +135,7 @@ type cseClusterSettingsInternal struct { SshPublicKey string PodCidr string ServiceCidr string - MachineHealthCheckEnabled bool AutoRepairOnErrors bool - VcdKeConfig vcdKeConfig } // tkgVersionBundle is a type that contains all the versions of the components of diff --git a/govcd/cse_util.go b/govcd/cse_util.go index b47553433..70c514419 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -682,8 +682,7 @@ func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClus } } - output.MachineHealthCheckEnabled = input.NodeHealthCheck - vcdKeConfig, err := getVcdKeConfig(org.client, cseComponentsVersions.VcdKeConfigRdeTypeVersion) + vcdKeConfig, err := getVcdKeConfig(org.client, cseComponentsVersions.VcdKeConfigRdeTypeVersion, input.NodeHealthCheck) if err != nil { return nil, err } @@ -805,7 +804,7 @@ func (tkgVersions tkgVersionBundle) kubernetesVersionIsOneMinorHigher(kubernetes // getVcdKeConfig gets the required information from the CSE Server configuration RDE (VCDKEConfig), such as the // Machine Health Check settings and the Container Registry URL. -func getVcdKeConfig(client *Client, vcdKeConfigVersion string) (*vcdKeConfig, error) { +func getVcdKeConfig(client *Client, vcdKeConfigVersion string, retrieveMachineHealtchCheckInfo bool) (*vcdKeConfig, error) { rdes, err := getRdesByName(client, "vmware", "VCDKEConfig", vcdKeConfigVersion, "vcdKeConfig") if err != nil { return nil, err @@ -827,15 +826,17 @@ func getVcdKeConfig(client *Client, vcdKeConfigVersion string) (*vcdKeConfig, er // https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.2/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.2/GUID-B5C19221-2ECA-4DCD-8EA1-8E391F6217C1.html result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"]) - mhc, ok := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"] - if !ok { - // If there is no "mhc" entry in the VCDKEConfig JSON, we skip setting this part of the Kubernetes cluster configuration - return result, nil + if retrieveMachineHealtchCheckInfo { + mhc, ok := profiles[0].(map[string]interface{})["K8Config"].(map[string]interface{})["mhc"] + if !ok { + // If there is no "mhc" entry in the VCDKEConfig JSON, we skip setting this part of the Kubernetes cluster configuration + return result, nil + } + result.MaxUnhealthyNodesPercentage = mhc.(map[string]interface{})["maxUnhealthyNodes"].(float64) + result.NodeStartupTimeout = mhc.(map[string]interface{})["nodeStartupTimeout"].(string) + result.NodeNotReadyTimeout = mhc.(map[string]interface{})["nodeUnknownTimeout"].(string) + result.NodeUnknownTimeout = mhc.(map[string]interface{})["nodeNotReadyTimeout"].(string) } - result.MaxUnhealthyNodesPercentage = mhc.(map[string]interface{})["maxUnhealthyNodes"].(float64) - result.NodeStartupTimeout = mhc.(map[string]interface{})["nodeStartupTimeout"].(string) - result.NodeNotReadyTimeout = mhc.(map[string]interface{})["nodeUnknownTimeout"].(string) - result.NodeUnknownTimeout = mhc.(map[string]interface{})["nodeNotReadyTimeout"].(string) return result, nil } diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 29a421d51..8b4af3cb2 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -2,6 +2,7 @@ package govcd import ( "fmt" + semver "github.com/hashicorp/go-version" "github.com/vmware/go-vcloud-director/v2/types/v56" "sigs.k8s.io/yaml" "strings" @@ -82,11 +83,11 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) } if input.NodeHealthCheck != nil { - vcdKeConfig, err := getVcdKeConfig(cluster.client, cluster.capvcdType.Status.VcdKe.VcdKeVersion) + vcdKeConfig, err := getVcdKeConfig(cluster.client, cluster.capvcdType.Status.VcdKe.VcdKeVersion, *input.NodeHealthCheck) if err != nil { return "", err } - err = cseUpdateNodeHealthCheckInYaml(yamlDocs, vcdKeConfig, cluster.NodeHealthCheck) + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, cluster.Name, cluster.CseVersion, vcdKeConfig) if err != nil { return "", err } @@ -277,32 +278,49 @@ func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernete // cseUpdateNodeHealthCheckInYaml updates the Kubernetes cluster described in the given YAML documents by adding or removing // the MachineHealthCheck object. // NOTE: This function doesn't modify the input, but returns a copy of the YAML with the modifications. -func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, vcdKeConfig *vcdKeConfig, mhcEnabled bool) error { - for _, d := range yamlDocuments { - if d["kind"] != "MachineHealthCheck" { - continue +func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, clusterName string, cseVersion semver.Version, vcdKeConfig *vcdKeConfig) ([]map[string]interface{}, error) { + mhcPosition := -1 + result := make([]map[string]interface{}, len(yamlDocuments)) + for i, d := range yamlDocuments { + if d["kind"] == "MachineHealthCheck" { + mhcPosition = i } + result[i] = d + } - maxUnhealthy := vcdKeConfig.MaxUnhealthyNodesPercentage - if !mhcEnabled { - maxUnhealthy = 0 + if mhcPosition < 0 { + // There is no MachineHealthCheck block + if vcdKeConfig == nil { + // We don't want it neither, so nothing to do + return result, nil } - // Replace them in the YAML - d["spec"].(map[string]interface{})["maxUnhealthy"] = fmt.Sprintf("%.0f%%", maxUnhealthy) - d["spec"].(map[string]interface{})["nodeStartupTimeout"] = fmt.Sprintf("%ss", strings.ReplaceAll(vcdKeConfig.NodeStartupTimeout, "s", "")) - unhealthyConditions := traverseMapAndGet[[]interface{}](d, "spec.unhealthyConditions") - for _, uc := range unhealthyConditions { - ucBlock := uc.(map[string]interface{}) - if ucBlock["status"] == "Unknown" { - ucBlock["timeout"] = fmt.Sprintf("%ss", strings.ReplaceAll(vcdKeConfig.NodeUnknownTimeout, "s", "")) - } - if ucBlock["status"] == "\"Ready\"" { - ucBlock["timeout"] = fmt.Sprintf("%ss", strings.ReplaceAll(vcdKeConfig.NodeNotReadyTimeout, "s", "")) - } + // We need to add the block to the slice of YAML documents + settings := &cseClusterSettingsInternal{CseVersion: cseVersion, Name: clusterName, VcdKeConfig: *vcdKeConfig} + mhcYaml, err := settings.generateMachineHealthCheckYaml() + if err != nil { + return nil, err } + var mhc map[string]interface{} + err = yaml.Unmarshal([]byte(mhcYaml), &mhc) + if err != nil { + return nil, err + } + result = append(result, mhc) + } else { + // There is a MachineHealthCheck block + if vcdKeConfig != nil { + // We want it, but it is already there, so nothing to do + // TODO: What happens in UI if the VCDKEConfig MHC values are changed, does it get reflected in the cluster? + // If that's the case, we might need to update this value always + return result, nil + } + + // We don't want Machine Health Checks, we delete the YAML document + result[mhcPosition] = result[len(result)-1] // We override the MachineHealthCheck block with the last document + result = result[:len(result)-1] // We remove the last document (now duplicated) } - return nil + return result, nil } // marshalMultipleYamlDocuments takes a slice of maps representing multiple YAML documents (one per item in the slice) and diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 7ff0a738b..677145960 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -3,6 +3,7 @@ package govcd import ( + semver "github.com/hashicorp/go-version" "github.com/vmware/go-vcloud-director/v2/types/v56" "os" "reflect" @@ -262,76 +263,53 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { t.Fatal("could not find the cluster name in the CAPI YAML test file") } + v, err := semver.NewVersion("4.1") + if err != nil { + t.Fatalf("incorrect version: %s", err) + } + // Deactivates Machine Health Check - err = cseUpdateNodeHealthCheckInYaml(yamlDocs, &vcdKeConfig{ - MaxUnhealthyNodesPercentage: 66, - NodeStartupTimeout: "100s", - NodeNotReadyTimeout: "200s", - NodeUnknownTimeout: "300s", - }, false) + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, nil) if err != nil { t.Fatalf("%s", err) } + // The resulting documents should not have that document for _, document := range yamlDocs { if document["kind"] == "MachineHealthCheck" { - if traverseMapAndGet[string](document, "spec.maxUnhealthy") != "0%" { - t.Fatalf("expected spec.maxUnhealthy to be updated to 0%%") - } - if traverseMapAndGet[string](document, "spec.nodeStartupTimeout") != "100s" { - t.Fatalf("expected spec.nodeStartupTimeout to remain at 100s") - } - unhealthyConditions := traverseMapAndGet[[]interface{}](document, "spec.unhealthyConditions") - for i, uc := range unhealthyConditions { - ucBlock := uc.(map[string]interface{}) - if ucBlock["status"] == "Unknown" { - if ucBlock["timeout"] != "300s" { - t.Fatalf("expected spec.unhealthyConditions[%d].timeout of status unknown to remain at 300s", i) - } - } - if ucBlock["status"] == "\"Ready\"" { - if ucBlock["timeout"] != "200s" { - t.Fatalf("expected spec.unhealthyConditions[%d].timeout to status ready to remain at 200s", i) - } - } - } + t.Fatal("Expected the MachineHealthCheck to be deleted, but it is there") } } // Enables Machine Health Check - err = cseUpdateNodeHealthCheckInYaml(yamlDocs, &vcdKeConfig{ - MaxUnhealthyNodesPercentage: 66, - NodeStartupTimeout: "100s", - NodeNotReadyTimeout: "200s", - NodeUnknownTimeout: "300s", - }, true) + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, &vcdKeConfig{ + MaxUnhealthyNodesPercentage: 12, + NodeStartupTimeout: "34", + NodeNotReadyTimeout: "56", + NodeUnknownTimeout: "78", + }) if err != nil { t.Fatalf("%s", err) } + // The resulting documents should have a MachineHealthCheck + found := false for _, document := range yamlDocs { - if document["kind"] == "MachineHealthCheck" { - if traverseMapAndGet[string](document, "spec.maxUnhealthy") != "66%" { - t.Fatalf("expected spec.maxUnhealthy to be updated to 66%%") - } - if traverseMapAndGet[string](document, "spec.nodeStartupTimeout") != "100s" { - t.Fatalf("expected spec.nodeStartupTimeout to remain at 100s") - } - unhealthyConditions := traverseMapAndGet[[]interface{}](document, "spec.unhealthyConditions") - for i, uc := range unhealthyConditions { - ucBlock := uc.(map[string]interface{}) - if ucBlock["status"] == "Unknown" { - if ucBlock["timeout"] != "300s" { - t.Fatalf("expected spec.unhealthyConditions[%d].timeout of status unknown to remain at 300s", i) - } - } - if ucBlock["status"] == "\"Ready\"" { - if ucBlock["timeout"] != "200s" { - t.Fatalf("expected spec.unhealthyConditions[%d].timeout to status ready to remain at 200s", i) - } - } - } + if document["kind"] != "MachineHealthCheck" { + continue } + maxUnhealthy := traverseMapAndGet[string](document, "spec.maxUnhealthy") + if maxUnhealthy != "12%" { + t.Fatalf("expected a 'spec.maxUnhealthy' = 12%%, but got %s", maxUnhealthy) + } + nodeStartupTimeout := traverseMapAndGet[string](document, "spec.nodeStartupTimeout") + if nodeStartupTimeout != "34s" { + t.Fatalf("expected a 'spec.nodeStartupTimeout' = 34s, but got %s", nodeStartupTimeout) + } + found = true + } + if !found { + t.Fatalf("expected a MachineHealthCheck block but got nothing") } } From 74301cfd0bbb2b17f1dc43e84f8bcd899d5e3dff Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 15:16:26 +0100 Subject: [PATCH 060/115] revert Signed-off-by: abarreiro --- govcd/cse/4.2.0/capiyaml_mhc.tmpl | 4 ++-- govcd/cse_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/govcd/cse/4.2.0/capiyaml_mhc.tmpl b/govcd/cse/4.2.0/capiyaml_mhc.tmpl index 3ff618dfb..d31e4c3ec 100644 --- a/govcd/cse/4.2.0/capiyaml_mhc.tmpl +++ b/govcd/cse/4.2.0/capiyaml_mhc.tmpl @@ -1,4 +1,4 @@ -apiVersion: cluster.x-k8s.io/v1beta2 +apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck metadata: name: "{{.ClusterName}}" @@ -19,4 +19,4 @@ spec: timeout: "{{.NodeUnknownTimeout}}" - type: Ready status: "False" - timeout: "{{.NodeNotReadyTimeout}}" + timeout: "{{.NodeNotReadyTimeout}}" \ No newline at end of file diff --git a/govcd/cse_test.go b/govcd/cse_test.go index ddcaf8653..71ba5dd2c 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -217,7 +217,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { } func (vcd *TestVCD) Test_Deleteme(check *C) { - cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:75e996ac-fe91-49b9-8e02-73759d0c8d8a") + cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:fd8e63dc-9127-407f-bf7b-29357442b8b4") check.Assert(err, IsNil) ovas, err := cluster.GetSupportedUpgrades(true) From d2e7f48545ce2ed1a03969a67b9f0a92fa477653 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 16:20:41 +0100 Subject: [PATCH 061/115] Fixes Signed-off-by: abarreiro --- govcd/cse.go | 16 +++++++--------- govcd/cse_test.go | 2 +- govcd/cse_util.go | 30 ++++++++++++++---------------- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index cd24a2e2f..b5c24005c 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -184,12 +184,12 @@ func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpd // As retrieving all OVAs one by one from VCD is expensive, the first time this method is called the returned OVAs are // cached to avoid querying VCD again multiple times. // If refreshOvas=true, this cache is cleared out and this method will query VCD for every vApp Template again. -// Therefore, the refreshOvas flag should be set to true only when VCD has new OVAs that need to be considered. +// Therefore, the refreshOvas flag should be set to true only when VCD has new OVAs that need to be considered or after a cluster upgrade. func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refreshOvas bool) ([]*types.VAppTemplate, error) { if refreshOvas { - cluster.supportedUpgrades = nil + cluster.supportedUpgrades = make([]*types.VAppTemplate, 0) } - if len(cluster.supportedUpgrades) != 0 { + if len(cluster.supportedUpgrades) > 0 { return cluster.supportedUpgrades, nil } @@ -197,7 +197,6 @@ func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refreshOvas bool) ([]* if err != nil { return nil, fmt.Errorf("could not get vApp Templates: %s", err) } - var tkgmOvas []*types.VAppTemplate for _, template := range vAppTemplates { // We can only know if the vApp Template is a TKGm OVA by inspecting its internals, hence we need to retrieve every one // of them one by one. This is an expensive operation, hence the cache. @@ -210,11 +209,10 @@ func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refreshOvas bool) ([]* continue // This means it's not a TKGm OVA, or it is not supported, so we skip it } if targetVersions.compareTkgVersion(cluster.TkgVersion.String()) == 1 && targetVersions.kubernetesVersionIsOneMinorHigher(cluster.KubernetesVersion.String()) { - tkgmOvas = append(tkgmOvas, vAppTemplate.VAppTemplate) + cluster.supportedUpgrades = append(cluster.supportedUpgrades, vAppTemplate.VAppTemplate) } } - cluster.supportedUpgrades = tkgmOvas - return tkgmOvas, nil + return cluster.supportedUpgrades, nil } // UpgradeCluster executes an update on the receiver cluster to upgrade the Kubernetes template of the cluster. @@ -253,10 +251,10 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh } } - if cluster.capvcdType.Status.VcdKe.State == "" { + if cluster.State == "" { return fmt.Errorf("can't update a Kubernetes cluster that does not have any state") } - if cluster.capvcdType.Status.VcdKe.State != "provisioned" { + if cluster.State != "provisioned" { return fmt.Errorf("can't update a Kubernetes cluster that is not in 'provisioned' state, as it is in '%s'", cluster.capvcdType.Status.VcdKe.State) } diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 71ba5dd2c..fcdfa8aa1 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -217,7 +217,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { } func (vcd *TestVCD) Test_Deleteme(check *C) { - cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:fd8e63dc-9127-407f-bf7b-29357442b8b4") + cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:7a09242a-ba6a-41d3-b918-bd3132f7f270") check.Assert(err, IsNil) ovas, err := cluster.GetSupportedUpgrades(true) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 70c514419..23b3cb1cf 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -76,6 +76,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu Events: make([]CseClusterEvent, 0), client: rde.client, capvcdType: capvcd, + supportedUpgrades: make([]*types.VAppTemplate, 0), } // Add all events to the resulting cluster @@ -163,22 +164,6 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu return result.Events[i].OccurredAt.After(result.Events[j].OccurredAt) }) - if capvcd.Status.Capvcd.Upgrade.Current.KubernetesVersion != "" { - version, err := semver.NewVersion(capvcd.Status.Capvcd.Upgrade.Current.KubernetesVersion) - if err != nil { - return nil, fmt.Errorf("could not read Kubernetes version: %s", err) - } - result.KubernetesVersion = *version - } - - if capvcd.Status.Capvcd.Upgrade.Current.TkgVersion != "" { - version, err := semver.NewVersion(capvcd.Status.Capvcd.Upgrade.Current.TkgVersion) - if err != nil { - return nil, fmt.Errorf("could not read Tkg version: %s", err) - } - result.TkgVersion = *version - } - if capvcd.Status.Capvcd.CapvcdVersion != "" { version, err := semver.NewVersion(capvcd.Status.Capvcd.CapvcdVersion) if err != nil { @@ -337,6 +322,13 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu if len(keys) > 0 { result.SshPublicKey = keys[0] // Optional field } + + version, err := semver.NewVersion(traverseMapAndGet[string](yamlDocument, "spec.version")) + if err != nil { + return nil, fmt.Errorf("could not read Kubernetes version: %s", err) + } + result.KubernetesVersion = *version + case "VCDMachineTemplate": name := traverseMapAndGet[string](yamlDocument, "metadata.name") sizingPolicyName := traverseMapAndGet[string](yamlDocument, "spec.template.spec.sizingPolicy") @@ -417,6 +409,12 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu case "VCDCluster": result.VirtualIpSubnet = traverseMapAndGet[string](yamlDocument, "spec.loadBalancerConfigSpec.vipSubnet") case "Cluster": + version, err := semver.NewVersion(traverseMapAndGet[string](yamlDocument, "metadata.annotations.TKGVERSION")) + if err != nil { + return nil, fmt.Errorf("could not read TKG version: %s", err) + } + result.TkgVersion = *version + cidrBlocks := traverseMapAndGet[[]interface{}](yamlDocument, "spec.clusterNetwork.pods.cidrBlocks") if len(cidrBlocks) == 0 { return nil, fmt.Errorf("expected at least one 'spec.clusterNetwork.pods.cidrBlocks' item") From 2eadf7b32fcf65b192b448413676bdf08a79daca Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 16:34:32 +0100 Subject: [PATCH 062/115] Fixes Signed-off-by: abarreiro --- govcd/cse_test.go | 5 +---- govcd/cse_util.go | 16 +++++++--------- govcd/cse_yaml.go | 17 ++++++++++++----- govcd/cse_yaml_unit_test.go | 4 ++-- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index fcdfa8aa1..f424b0c60 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -220,9 +220,6 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:7a09242a-ba6a-41d3-b918-bd3132f7f270") check.Assert(err, IsNil) - ovas, err := cluster.GetSupportedUpgrades(true) - check.Assert(err, IsNil) - - err = cluster.UpgradeCluster(ovas[0].ID, true) + err = cluster.SetHealthCheck(false, true) check.Assert(err, IsNil) } diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 23b3cb1cf..10e34ac4b 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -684,9 +684,7 @@ func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClus if err != nil { return nil, err } - if vcdKeConfig != nil { - output.VcdKeConfig = *vcdKeConfig - } + output.VcdKeConfig = vcdKeConfig output.Owner = input.Owner if input.Owner == "" { @@ -802,24 +800,24 @@ func (tkgVersions tkgVersionBundle) kubernetesVersionIsOneMinorHigher(kubernetes // getVcdKeConfig gets the required information from the CSE Server configuration RDE (VCDKEConfig), such as the // Machine Health Check settings and the Container Registry URL. -func getVcdKeConfig(client *Client, vcdKeConfigVersion string, retrieveMachineHealtchCheckInfo bool) (*vcdKeConfig, error) { +func getVcdKeConfig(client *Client, vcdKeConfigVersion string, retrieveMachineHealtchCheckInfo bool) (vcdKeConfig, error) { + result := vcdKeConfig{} rdes, err := getRdesByName(client, "vmware", "VCDKEConfig", vcdKeConfigVersion, "vcdKeConfig") if err != nil { - return nil, err + return result, err } if len(rdes) != 1 { - return nil, fmt.Errorf("expected exactly one VCDKEConfig RDE with version '%s', but got %d", vcdKeConfigVersion, len(rdes)) + return result, fmt.Errorf("expected exactly one VCDKEConfig RDE with version '%s', but got %d", vcdKeConfigVersion, len(rdes)) } profiles, ok := rdes[0].DefinedEntity.Entity["profiles"].([]interface{}) if !ok { - return nil, fmt.Errorf("wrong format of VCDKEConfig RDE contents, expected a 'profiles' array") + return result, fmt.Errorf("wrong format of VCDKEConfig RDE contents, expected a 'profiles' array") } if len(profiles) == 0 { - return nil, fmt.Errorf("wrong format of VCDKEConfig RDE contents, expected a non-empty 'profiles' element") + return result, fmt.Errorf("wrong format of VCDKEConfig RDE contents, expected a non-empty 'profiles' element") } - result := &vcdKeConfig{} // We append /tkg as required, even in air-gapped environments: // https://docs.vmware.com/en/VMware-Cloud-Director-Container-Service-Extension/4.2/VMware-Cloud-Director-Container-Service-Extension-Install-provider-4.2/GUID-B5C19221-2ECA-4DCD-8EA1-8E391F6217C1.html result.ContainerRegistryUrl = fmt.Sprintf("%s/tkg", profiles[0].(map[string]interface{})["containerRegistryUrl"]) diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 8b4af3cb2..06cad592f 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -83,7 +83,11 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) } if input.NodeHealthCheck != nil { - vcdKeConfig, err := getVcdKeConfig(cluster.client, cluster.capvcdType.Status.VcdKe.VcdKeVersion, *input.NodeHealthCheck) + cseComponentsVersions, err := getCseComponentsVersions(cluster.CseVersion) + if err != nil { + return "", err + } + vcdKeConfig, err := getVcdKeConfig(cluster.client, cseComponentsVersions.VcdKeConfigRdeTypeVersion, *input.NodeHealthCheck) if err != nil { return "", err } @@ -278,7 +282,7 @@ func cseAddWorkerPoolsInYaml(docs []map[string]interface{}, cluster CseKubernete // cseUpdateNodeHealthCheckInYaml updates the Kubernetes cluster described in the given YAML documents by adding or removing // the MachineHealthCheck object. // NOTE: This function doesn't modify the input, but returns a copy of the YAML with the modifications. -func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, clusterName string, cseVersion semver.Version, vcdKeConfig *vcdKeConfig) ([]map[string]interface{}, error) { +func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, clusterName string, cseVersion semver.Version, vcdKeConfig vcdKeConfig) ([]map[string]interface{}, error) { mhcPosition := -1 result := make([]map[string]interface{}, len(yamlDocuments)) for i, d := range yamlDocuments { @@ -288,15 +292,18 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, clus result[i] = d } + machineHealthCheckEnabled := vcdKeConfig.NodeUnknownTimeout != "" && vcdKeConfig.NodeStartupTimeout != "" && vcdKeConfig.NodeNotReadyTimeout != "" && + vcdKeConfig.MaxUnhealthyNodesPercentage != 0 + if mhcPosition < 0 { // There is no MachineHealthCheck block - if vcdKeConfig == nil { + if !machineHealthCheckEnabled { // We don't want it neither, so nothing to do return result, nil } // We need to add the block to the slice of YAML documents - settings := &cseClusterSettingsInternal{CseVersion: cseVersion, Name: clusterName, VcdKeConfig: *vcdKeConfig} + settings := &cseClusterSettingsInternal{CseVersion: cseVersion, Name: clusterName, VcdKeConfig: vcdKeConfig} mhcYaml, err := settings.generateMachineHealthCheckYaml() if err != nil { return nil, err @@ -309,7 +316,7 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, clus result = append(result, mhc) } else { // There is a MachineHealthCheck block - if vcdKeConfig != nil { + if machineHealthCheckEnabled { // We want it, but it is already there, so nothing to do // TODO: What happens in UI if the VCDKEConfig MHC values are changed, does it get reflected in the cluster? // If that's the case, we might need to update this value always diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 677145960..24b321966 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -269,7 +269,7 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { } // Deactivates Machine Health Check - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, nil) + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, vcdKeConfig{}) if err != nil { t.Fatalf("%s", err) } @@ -282,7 +282,7 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { } // Enables Machine Health Check - yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, &vcdKeConfig{ + yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, vcdKeConfig{ MaxUnhealthyNodesPercentage: 12, NodeStartupTimeout: "34", NodeNotReadyTimeout: "56", From 859bca0c642727664b4ce95202499f4b36e1bae8 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 17:51:48 +0100 Subject: [PATCH 063/115] Changelog Signed-off-by: abarreiro --- .changes/v2.23.0/645-features.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .changes/v2.23.0/645-features.md diff --git a/.changes/v2.23.0/645-features.md b/.changes/v2.23.0/645-features.md new file mode 100644 index 000000000..7fdeedbcf --- /dev/null +++ b/.changes/v2.23.0/645-features.md @@ -0,0 +1,13 @@ +* Added a new type `CseKubernetesCluster` to manage Container Service Extension Kubernetes clusters [GH-645] +* Added new methods `Org.CseCreateKubernetesCluster` and `Org.CseCreateKubernetesClusterAsync` to create Kubernetes clusters + in a VCD with Container Service Extension installed [GH-645] +* Added a new method `CseKubernetesCluster.GetKubeconfig` to retrieve the *kubeconfig* from a provisioned Container Service Extension Kubernetes cluster [GH-645] +* Added a new method `CseKubernetesCluster.Refresh` to refresh a Container Service Extension Kubernetes cluster [GH-645] +* Added new methods to update a Container Service Extension Kubernetes cluster: `CseKubernetesCluster.UpdateWorkerPools`, + `CseKubernetesCluster.AddWorkerPools`, `CseKubernetesCluster.UpdateControlPlane`, `CseKubernetesCluster.UpgradeCluster`, + `CseKubernetesCluster.SetHealthCheck` and `CseKubernetesCluster.SetAutoRepairOnErrors` [GH-645] +* Added a new method `CseKubernetesCluster.GetSupportedUpgrades` to retrieve all the valid TKGm OVAs that a given Container Service Extension Kubernetes cluster + can use to be upgraded [GH-645] +* Added a new method `CseKubernetesCluster.Delete` to delete a cluster [GH-645] +* Added new types `CseClusterSettings`, `CseControlPlaneSettings`, `CseWorkerPoolSettings` and `CseDefaultStorageClassSettings` to configure the Container Service Extension Kubernetes clusters creation process [GH-645] +* Added new types `CseClusterUpdateInput`, `CseControlPlaneUpdateInput` and `CseWorkerPoolUpdateInput` to configure the Container Service Extension Kubernetes clusters update process [GH-645] From 31038d4786da60547dc6fe2b52388a87438f7ab3 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 17:54:35 +0100 Subject: [PATCH 064/115] Fix wrong replace Signed-off-by: abarreiro --- govcd/access_control.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/govcd/access_control.go b/govcd/access_control.go index edeea4a47..140722e4f 100644 --- a/govcd/access_control.go +++ b/govcd/access_control.go @@ -15,7 +15,7 @@ import ( "github.com/vmware/go-vcloud-director/v2/types/v56" ) -// orgInfoCache is a nameToIdCache to save org information, avoid repeated calls to compute the same result. +// orgInfoCache is a cache to save org information, avoid repeated calls to compute the same result. // The keys to this map are the requesting objects IDs. var orgInfoCache = make(map[string]*TenantContext) From 3f212607fea9c6cfcb91a4aacecdb1d5d4028f88 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 17:57:10 +0100 Subject: [PATCH 065/115] Fix wrong replace Signed-off-by: abarreiro --- .changes/v2.23.0/645-features.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.changes/v2.23.0/645-features.md b/.changes/v2.23.0/645-features.md index 7fdeedbcf..201def50f 100644 --- a/.changes/v2.23.0/645-features.md +++ b/.changes/v2.23.0/645-features.md @@ -1,6 +1,7 @@ * Added a new type `CseKubernetesCluster` to manage Container Service Extension Kubernetes clusters [GH-645] * Added new methods `Org.CseCreateKubernetesCluster` and `Org.CseCreateKubernetesClusterAsync` to create Kubernetes clusters - in a VCD with Container Service Extension installed [GH-645] + in a VCD appliance with Container Service Extension installed [GH-645] +* Added new methods `VCDClient.CseGetKubernetesClusterById` and `Org.CseGetKubernetesClustersByName` to retrieve a Container Service Extension Kubernetes cluster [GH-645] * Added a new method `CseKubernetesCluster.GetKubeconfig` to retrieve the *kubeconfig* from a provisioned Container Service Extension Kubernetes cluster [GH-645] * Added a new method `CseKubernetesCluster.Refresh` to refresh a Container Service Extension Kubernetes cluster [GH-645] * Added new methods to update a Container Service Extension Kubernetes cluster: `CseKubernetesCluster.UpdateWorkerPools`, From beaa5b9eccea98ba01e751c56ff1fae1e5ada6b0 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 17:58:12 +0100 Subject: [PATCH 066/115] # Signed-off-by: abarreiro --- .changes/v2.23.0/645-features.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.changes/v2.23.0/645-features.md b/.changes/v2.23.0/645-features.md index 201def50f..24cbb1b53 100644 --- a/.changes/v2.23.0/645-features.md +++ b/.changes/v2.23.0/645-features.md @@ -1,14 +1,19 @@ * Added a new type `CseKubernetesCluster` to manage Container Service Extension Kubernetes clusters [GH-645] * Added new methods `Org.CseCreateKubernetesCluster` and `Org.CseCreateKubernetesClusterAsync` to create Kubernetes clusters in a VCD appliance with Container Service Extension installed [GH-645] -* Added new methods `VCDClient.CseGetKubernetesClusterById` and `Org.CseGetKubernetesClustersByName` to retrieve a Container Service Extension Kubernetes cluster [GH-645] -* Added a new method `CseKubernetesCluster.GetKubeconfig` to retrieve the *kubeconfig* from a provisioned Container Service Extension Kubernetes cluster [GH-645] -* Added a new method `CseKubernetesCluster.Refresh` to refresh a Container Service Extension Kubernetes cluster [GH-645] +* Added new methods `VCDClient.CseGetKubernetesClusterById` and `Org.CseGetKubernetesClustersByName` to retrieve a + Container Service Extension Kubernetes cluster [GH-645] +* Added a new method `CseKubernetesCluster.GetKubeconfig` to retrieve the *kubeconfig* of a provisioned Container Service + Extension Kubernetes cluster [GH-645] +* Added a new method `CseKubernetesCluster.Refresh` to refresh the information and properties of an existing Container + Service Extension Kubernetes cluster [GH-645] * Added new methods to update a Container Service Extension Kubernetes cluster: `CseKubernetesCluster.UpdateWorkerPools`, `CseKubernetesCluster.AddWorkerPools`, `CseKubernetesCluster.UpdateControlPlane`, `CseKubernetesCluster.UpgradeCluster`, `CseKubernetesCluster.SetHealthCheck` and `CseKubernetesCluster.SetAutoRepairOnErrors` [GH-645] -* Added a new method `CseKubernetesCluster.GetSupportedUpgrades` to retrieve all the valid TKGm OVAs that a given Container Service Extension Kubernetes cluster - can use to be upgraded [GH-645] +* Added a new method `CseKubernetesCluster.GetSupportedUpgrades` to retrieve all the valid TKGm OVAs that a given Container + Service Extension Kubernetes cluster can use to be upgraded [GH-645] * Added a new method `CseKubernetesCluster.Delete` to delete a cluster [GH-645] -* Added new types `CseClusterSettings`, `CseControlPlaneSettings`, `CseWorkerPoolSettings` and `CseDefaultStorageClassSettings` to configure the Container Service Extension Kubernetes clusters creation process [GH-645] -* Added new types `CseClusterUpdateInput`, `CseControlPlaneUpdateInput` and `CseWorkerPoolUpdateInput` to configure the Container Service Extension Kubernetes clusters update process [GH-645] +* Added new types `CseClusterSettings`, `CseControlPlaneSettings`, `CseWorkerPoolSettings` and `CseDefaultStorageClassSettings` + to configure the Container Service Extension Kubernetes clusters creation process [GH-645] +* Added new types `CseClusterUpdateInput`, `CseControlPlaneUpdateInput` and `CseWorkerPoolUpdateInput` to configure the + Container Service Extension Kubernetes clusters update process [GH-645] From 7b77a925f3af7719c8436cc8c22cb5acb7f4f07f Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 16 Feb 2024 18:00:00 +0100 Subject: [PATCH 067/115] # Signed-off-by: abarreiro --- .changes/v2.23.0/645-features.md | 2 +- govcd/cse.go | 4 ++-- govcd/cse_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.changes/v2.23.0/645-features.md b/.changes/v2.23.0/645-features.md index 24cbb1b53..3b4537c2b 100644 --- a/.changes/v2.23.0/645-features.md +++ b/.changes/v2.23.0/645-features.md @@ -9,7 +9,7 @@ Service Extension Kubernetes cluster [GH-645] * Added new methods to update a Container Service Extension Kubernetes cluster: `CseKubernetesCluster.UpdateWorkerPools`, `CseKubernetesCluster.AddWorkerPools`, `CseKubernetesCluster.UpdateControlPlane`, `CseKubernetesCluster.UpgradeCluster`, - `CseKubernetesCluster.SetHealthCheck` and `CseKubernetesCluster.SetAutoRepairOnErrors` [GH-645] + `CseKubernetesCluster.SetNodeHealthCheck` and `CseKubernetesCluster.SetAutoRepairOnErrors` [GH-645] * Added a new method `CseKubernetesCluster.GetSupportedUpgrades` to retrieve all the valid TKGm OVAs that a given Container Service Extension Kubernetes cluster can use to be upgraded [GH-645] * Added a new method `CseKubernetesCluster.Delete` to delete a cluster [GH-645] diff --git a/govcd/cse.go b/govcd/cse.go index b5c24005c..6135765f5 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -224,9 +224,9 @@ func (cluster *CseKubernetesCluster) UpgradeCluster(kubernetesTemplateOvaId stri }, refresh) } -// SetHealthCheck executes an update on the receiver cluster to enable or disable the machine health check capabilities. +// SetNodeHealthCheck executes an update on the receiver cluster to enable or disable the machine health check capabilities. // If refresh=true, it retrieves the latest state of the cluster from VCD before updating. -func (cluster *CseKubernetesCluster) SetHealthCheck(healthCheckEnabled bool, refresh bool) error { +func (cluster *CseKubernetesCluster) SetNodeHealthCheck(healthCheckEnabled bool, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ NodeHealthCheck: &healthCheckEnabled, }, refresh) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index f424b0c60..1437df032 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -220,6 +220,6 @@ func (vcd *TestVCD) Test_Deleteme(check *C) { cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:7a09242a-ba6a-41d3-b918-bd3132f7f270") check.Assert(err, IsNil) - err = cluster.SetHealthCheck(false, true) + err = cluster.SetNodeHealthCheck(false, true) check.Assert(err, IsNil) } From 5878f65920c68dc18a586270a3d00e0de4b0206e Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 20 Feb 2024 12:01:02 +0100 Subject: [PATCH 068/115] Add projector events Signed-off-by: abarreiro --- govcd/api_vcd_test.go | 15 ++++---- govcd/cse_test.go | 54 ++++++++++++++++++++++------- govcd/cse_util.go | 20 +++++++++++ govcd/sample_govcd_test_config.yaml | 2 ++ types/v56/cse.go | 1 + 5 files changed, 72 insertions(+), 20 deletions(-) diff --git a/govcd/api_vcd_test.go b/govcd/api_vcd_test.go index 7d3220636..4a0cd81b0 100644 --- a/govcd/api_vcd_test.go +++ b/govcd/api_vcd_test.go @@ -240,13 +240,14 @@ type TestConfig struct { UiPluginPath string `yaml:"uiPluginPath,omitempty"` } `yaml:"media"` Cse struct { - SolutionsOrg string `yaml:"solutionsOrg,omitempty"` - TenantOrg string `yaml:"tenantOrg,omitempty"` - TenantVdc string `yaml:"tenantVdc,omitempty"` - RoutedNetwork string `yaml:"routedNetwork,omitempty"` - EdgeGateway string `yaml:"edgeGateway,omitempty"` - OvaCatalog string `yaml:"ovaCatalog,omitempty"` - OvaName string `yaml:"ovaName,omitempty"` + SolutionsOrg string `yaml:"solutionsOrg,omitempty"` + TenantOrg string `yaml:"tenantOrg,omitempty"` + TenantVdc string `yaml:"tenantVdc,omitempty"` + RoutedNetwork string `yaml:"routedNetwork,omitempty"` + EdgeGateway string `yaml:"edgeGateway,omitempty"` + StorageProfile string `yaml:"storageProfile,omitempty"` + OvaCatalog string `yaml:"ovaCatalog,omitempty"` + OvaName string `yaml:"ovaName,omitempty"` } `yaml:"cse"` } diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 1437df032..f08d1a06a 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -26,7 +26,7 @@ func skipCseTests(testConfig TestConfig) bool { } return testConfig.Cse.SolutionsOrg == "" || testConfig.Cse.TenantOrg == "" || testConfig.Cse.OvaName == "" || testConfig.Cse.RoutedNetwork == "" || testConfig.Cse.EdgeGateway == "" || testConfig.Cse.OvaCatalog == "" || testConfig.Cse.TenantVdc == "" || - testConfig.VCD.StorageProfile.SP1 == "" + testConfig.Cse.StorageProfile == "" } // Test_Cse @@ -52,7 +52,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { net, err := vdc.GetOrgVdcNetworkByName(vcd.config.Cse.RoutedNetwork, false) check.Assert(err, IsNil) - sp, err := vdc.FindStorageProfileReference(vcd.config.VCD.StorageProfile.SP1) + sp, err := vdc.FindStorageProfileReference(vcd.config.Cse.StorageProfile) check.Assert(err, IsNil) policies, err := vcd.client.GetAllVdcComputePoliciesV2(url.Values{ @@ -61,14 +61,18 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(err, IsNil) check.Assert(len(policies), Equals, 1) - token, err := vcd.client.CreateToken(vcd.config.Provider.SysOrg, check.TestName()+"124") // TODO: Remove number suffix + token, err := vcd.client.CreateToken(vcd.config.Provider.SysOrg, check.TestName()) check.Assert(err, IsNil) + defer func() { + err = token.Delete() + check.Assert(err, IsNil) + }() AddToCleanupListOpenApi(token.Token.Name, check.TestName(), types.OpenApiPathVersion1_0_0+types.OpenApiEndpointTokens+token.Token.ID) apiToken, err := token.GetInitialApiToken() check.Assert(err, IsNil) - cseVersion, err := semver.NewVersion("4.1") + cseVersion, err := semver.NewVersion("4.2.0") check.Assert(err, IsNil) check.Assert(cseVersion, NotNil) @@ -185,13 +189,41 @@ func (vcd *TestVCD) Test_Cse(check *C) { allClusters, err := org.CseGetKubernetesClustersByName(clusterGet.CseVersion, clusterGet.Name) check.Assert(err, IsNil) check.Assert(len(allClusters), Equals, 1) - check.Assert(allClusters[0], DeepEquals, clusterGet) + check.Assert(cluster.CseVersion.String(), Equals, allClusters[0].CseVersion.String()) + check.Assert(cluster.Name, Equals, allClusters[0].Name) + check.Assert(cluster.OrganizationId, Equals, allClusters[0].OrganizationId) + check.Assert(cluster.VdcId, Equals, allClusters[0].VdcId) + check.Assert(cluster.NetworkId, Equals, allClusters[0].NetworkId) + check.Assert(cluster.KubernetesTemplateOvaId, Equals, allClusters[0].KubernetesTemplateOvaId) + check.Assert(cluster.ControlPlane, DeepEquals, allClusters[0].ControlPlane) + check.Assert(cluster.WorkerPools, DeepEquals, allClusters[0].WorkerPools) + check.Assert(cluster.DefaultStorageClass, NotNil) + check.Assert(*cluster.DefaultStorageClass, DeepEquals, *allClusters[0].DefaultStorageClass) + check.Assert(cluster.Owner, Equals, allClusters[0].Owner) + check.Assert(cluster.ApiToken, Not(Equals), allClusters[0].ApiToken) + check.Assert(allClusters[0].ApiToken, Equals, "******") // This one can't be recovered + check.Assert(cluster.NodeHealthCheck, Equals, allClusters[0].NodeHealthCheck) + check.Assert(cluster.PodCidr, Equals, allClusters[0].PodCidr) + check.Assert(cluster.ServiceCidr, Equals, allClusters[0].ServiceCidr) + check.Assert(cluster.SshPublicKey, Equals, allClusters[0].SshPublicKey) + check.Assert(cluster.VirtualIpSubnet, Equals, allClusters[0].VirtualIpSubnet) + check.Assert(cluster.AutoRepairOnErrors, Equals, allClusters[0].AutoRepairOnErrors) + check.Assert(cluster.VirtualIpSubnet, Equals, allClusters[0].VirtualIpSubnet) + check.Assert(cluster.ID, Equals, allClusters[0].ID) + check.Assert(allClusters[0].Etag, Not(Equals), "") + check.Assert(cluster.KubernetesVersion, Equals, allClusters[0].KubernetesVersion) + check.Assert(cluster.TkgVersion.String(), Equals, allClusters[0].TkgVersion.String()) + check.Assert(cluster.CapvcdVersion.String(), Equals, allClusters[0].CapvcdVersion.String()) + check.Assert(cluster.ClusterResourceSetBindings, DeepEquals, allClusters[0].ClusterResourceSetBindings) + check.Assert(cluster.CpiVersion.String(), Equals, allClusters[0].CpiVersion.String()) + check.Assert(cluster.CsiVersion.String(), Equals, allClusters[0].CsiVersion.String()) + check.Assert(cluster.State, Equals, allClusters[0].State) // Update worker pool from 1 node to 2 // Pre-check. This should be 1, as it was created with just 1 pool - for _, nodePool := range cluster.capvcdType.Status.Capvcd.NodePool { + for _, nodePool := range cluster.WorkerPools { if nodePool.Name == clusterSettings.WorkerPools[0].Name { - check.Assert(nodePool.DesiredReplicas, Equals, 1) + check.Assert(nodePool.MachineCount, Equals, 1) } } // Perform the update @@ -200,20 +232,16 @@ func (vcd *TestVCD) Test_Cse(check *C) { // Post-check. This should be 2, as it should have scaled up foundWorkerPool := false - for _, nodePool := range cluster.capvcdType.Status.Capvcd.NodePool { + for _, nodePool := range cluster.WorkerPools { if nodePool.Name == clusterSettings.WorkerPools[0].Name { foundWorkerPool = true - check.Assert(nodePool.DesiredReplicas, Equals, 2) + check.Assert(nodePool.MachineCount, Equals, 2) } } check.Assert(foundWorkerPool, Equals, true) err = cluster.Delete(0) check.Assert(err, IsNil) - - err = token.Delete() - check.Assert(err, IsNil) - } func (vcd *TestVCD) Test_Deleteme(check *C) { diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 10e34ac4b..7a1093573 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -160,6 +160,26 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu Details: s.AdditionalDetails.DetailedError, }) } + for _, s := range capvcd.Status.Projector.EventSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "event", + ResourceId: s.VcdResourceId, + ResourceName: s.VcdResourceName, + OccurredAt: s.OccurredAt, + Details: s.Name, + }) + } + for _, s := range capvcd.Status.Projector.ErrorSet { + result.Events = append(result.Events, CseClusterEvent{ + Name: s.Name, + Type: "error", + ResourceId: s.VcdResourceId, + ResourceName: s.VcdResourceName, + OccurredAt: s.OccurredAt, + Details: s.AdditionalDetails.DetailedError, + }) + } sort.SliceStable(result.Events, func(i, j int) bool { return result.Events[i].OccurredAt.After(result.Events[j].OccurredAt) }) diff --git a/govcd/sample_govcd_test_config.yaml b/govcd/sample_govcd_test_config.yaml index b4dc10354..10f923ac1 100644 --- a/govcd/sample_govcd_test_config.yaml +++ b/govcd/sample_govcd_test_config.yaml @@ -264,6 +264,8 @@ cse: routedNetwork: "tenant_net_routed" # The edge gateway which the Kubernetes clusters use edgeGateway: "tenant_edgegateway" + # The storage profile which the Kubernetes clusters use + storageProfile: "*" # The catalog which the Kubernetes clusters use ovaCatalog: "tkgm_catalog" # The TKGm OVA which the Kubernetes clusters use diff --git a/types/v56/cse.go b/types/v56/cse.go index 619fe5ce3..31e82b16f 100644 --- a/types/v56/cse.go +++ b/types/v56/cse.go @@ -198,6 +198,7 @@ type Capvcd struct { EventSet []struct { Name string `json:"name,omitempty"` OccurredAt time.Time `json:"occurredAt,omitempty"` + VcdResourceId string `json:"vcdResourceId,omitempty"` VcdResourceName string `json:"vcdResourceName,omitempty"` AdditionalDetails struct { Event string `json:"event,omitempty"` From 4422a7abf2842ed9688afdeeb851fbab54c11e7a Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 20 Feb 2024 12:40:31 +0100 Subject: [PATCH 069/115] Add projector events Signed-off-by: abarreiro --- govcd/cse_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 7a1093573..7ae0050ce 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -180,7 +180,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu Details: s.AdditionalDetails.DetailedError, }) } - sort.SliceStable(result.Events, func(i, j int) bool { + sort.Slice(result.Events, func(i, j int) bool { return result.Events[i].OccurredAt.After(result.Events[j].OccurredAt) }) From 7befc6982306edcb215a01a13ecca2008a57efbb Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 20 Feb 2024 13:04:49 +0100 Subject: [PATCH 070/115] test Signed-off-by: abarreiro --- govcd/cse_test.go | 2 +- govcd/cse_util.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index f08d1a06a..b53dc2390 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -245,7 +245,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { } func (vcd *TestVCD) Test_Deleteme(check *C) { - cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:7a09242a-ba6a-41d3-b918-bd3132f7f270") + cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:bac3333c-51ca-4b3e-a239-ae8669f899e6") check.Assert(err, IsNil) err = cluster.SetNodeHealthCheck(false, true) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 7ae0050ce..7a1093573 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -180,7 +180,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu Details: s.AdditionalDetails.DetailedError, }) } - sort.Slice(result.Events, func(i, j int) bool { + sort.SliceStable(result.Events, func(i, j int) bool { return result.Events[i].OccurredAt.After(result.Events[j].OccurredAt) }) From a9805422ac07011c0c290d009b7386a0fbc264b8 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 21 Feb 2024 09:11:45 +0100 Subject: [PATCH 071/115] Fix delete Signed-off-by: abarreiro --- govcd/cse.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 6135765f5..ee2813901 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -326,7 +326,8 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh func (cluster *CseKubernetesCluster) Delete(timeoutMinutes time.Duration) error { var elapsed time.Duration start := time.Now() - vcdKe := map[string]interface{}{} + markForDelete := false + forceDelete := false for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies timeoutMinutes=0, we wait forever rde, err := getRdeById(cluster.client, cluster.ID) if err != nil { @@ -336,14 +337,13 @@ func (cluster *CseKubernetesCluster) Delete(timeoutMinutes time.Duration) error return fmt.Errorf("could not retrieve the Kubernetes cluster with ID '%s': %s", cluster.ID, err) } - markForDelete := traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.markForDelete") - forceDelete := traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.forceDelete") + markForDelete = traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.markForDelete") + forceDelete = traverseMapAndGet[bool](rde.DefinedEntity.Entity, "spec.vcdKe.forceDelete") if !markForDelete || !forceDelete { // Mark the cluster for deletion - vcdKe["markForDelete"] = true - vcdKe["forceDelete"] = true - rde.DefinedEntity.Entity["spec"].(map[string]interface{})["vcdKe"] = vcdKe + rde.DefinedEntity.Entity["spec"].(map[string]interface{})["vcdKe"].(map[string]interface{})["markForDelete"] = true + rde.DefinedEntity.Entity["spec"].(map[string]interface{})["vcdKe"].(map[string]interface{})["forceDelete"] = true err = rde.Update(*rde.DefinedEntity) if err != nil { // We ignore any ETag error. This just means a clash with the CSE Server, we just try again @@ -359,7 +359,7 @@ func (cluster *CseKubernetesCluster) Delete(timeoutMinutes time.Duration) error } // We give a hint to the user about the deletion process result - if len(vcdKe) >= 2 && vcdKe["markForDelete"].(bool) && vcdKe["forceDelete"].(bool) { + if markForDelete && forceDelete { return fmt.Errorf("timeout of %v minutes reached, the cluster was successfully marked for deletion but was not removed in time", timeoutMinutes) } return fmt.Errorf("timeout of %v minutes reached, the cluster was not marked for deletion, please try again", timeoutMinutes) From 58d3e8997a7b14dc269c4dee512754f7106e0802 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 11:06:01 +0100 Subject: [PATCH 072/115] Improve tests and fix tiny bug Signed-off-by: abarreiro --- govcd/api_vcd_test.go | 6 +- govcd/cse.go | 3 +- govcd/cse_test.go | 280 ++++++++++++++++++---------- govcd/cse_yaml.go | 4 +- govcd/sample_govcd_test_config.yaml | 2 + 5 files changed, 196 insertions(+), 99 deletions(-) diff --git a/govcd/api_vcd_test.go b/govcd/api_vcd_test.go index 4a0cd81b0..6d8cec6d4 100644 --- a/govcd/api_vcd_test.go +++ b/govcd/api_vcd_test.go @@ -240,6 +240,7 @@ type TestConfig struct { UiPluginPath string `yaml:"uiPluginPath,omitempty"` } `yaml:"media"` Cse struct { + Version string `yaml:"version,omitempty"` SolutionsOrg string `yaml:"solutionsOrg,omitempty"` TenantOrg string `yaml:"tenantOrg,omitempty"` TenantVdc string `yaml:"tenantVdc,omitempty"` @@ -248,7 +249,10 @@ type TestConfig struct { StorageProfile string `yaml:"storageProfile,omitempty"` OvaCatalog string `yaml:"ovaCatalog,omitempty"` OvaName string `yaml:"ovaName,omitempty"` - } `yaml:"cse"` + } `yaml:"cse,omitempty"` + Dse struct { + IsoName string `yaml:"isoName,omitempty"` + } `yaml:"dse,omitempty"` } // Test struct for vcloud-director. diff --git a/govcd/cse.go b/govcd/cse.go index ee2813901..56b6fd651 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -77,7 +77,8 @@ func (vcdClient *VCDClient) CseGetKubernetesClusterById(id string) (*CseKubernet return getCseKubernetesClusterById(&vcdClient.Client, id) } -// CseGetKubernetesClustersByName retrieves the CSE Kubernetes cluster from VCD with the given name +// CseGetKubernetesClustersByName retrieves the CSE Kubernetes cluster from VCD with the given name. +// Note: The clusters retrieved won't have a valid ETag to perform operations on them. Use VCDClient.CseGetKubernetesClusterById for that instead. func (org *Org) CseGetKubernetesClustersByName(cseVersion semver.Version, name string) ([]*CseKubernetesCluster, error) { cseSubcomponents, err := getCseComponentsVersions(cseVersion) if err != nil { diff --git a/govcd/cse_test.go b/govcd/cse_test.go index b53dc2390..04463c4e0 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -24,17 +24,19 @@ func skipCseTests(testConfig TestConfig) bool { if cse := os.Getenv("TEST_VCD_CSE"); cse == "" { return true } - return testConfig.Cse.SolutionsOrg == "" || testConfig.Cse.TenantOrg == "" || testConfig.Cse.OvaName == "" || + return testConfig.Cse.Version == "" || testConfig.Cse.SolutionsOrg == "" || testConfig.Cse.TenantOrg == "" || testConfig.Cse.OvaName == "" || testConfig.Cse.RoutedNetwork == "" || testConfig.Cse.EdgeGateway == "" || testConfig.Cse.OvaCatalog == "" || testConfig.Cse.TenantVdc == "" || testConfig.Cse.StorageProfile == "" } -// Test_Cse +// Test_Cse tests all possible combinations of the CSE CRUD operations. func (vcd *TestVCD) Test_Cse(check *C) { if skipCseTests(vcd.config) { check.Skip(fmt.Sprintf(TestRequiresCseConfiguration, check.TestName())) } + // Prerequisites: We need to read several items before creating the cluster. + org, err := vcd.client.GetOrgByName(vcd.config.Cse.TenantOrg) check.Assert(err, IsNil) @@ -43,6 +45,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { ova, err := catalog.GetVAppTemplateByName(vcd.config.Cse.OvaName) check.Assert(err, IsNil) + tkgBundle, err := getTkgVersionBundleFromVAppTemplate(ova.VAppTemplate) check.Assert(err, IsNil) @@ -72,10 +75,14 @@ func (vcd *TestVCD) Test_Cse(check *C) { apiToken, err := token.GetInitialApiToken() check.Assert(err, IsNil) - cseVersion, err := semver.NewVersion("4.2.0") + cseVersion, err := semver.NewVersion(vcd.config.Cse.Version) check.Assert(err, IsNil) check.Assert(cseVersion, NotNil) + v411, err := semver.NewVersion("4.1.1") + check.Assert(err, IsNil) + + // Create the cluster clusterSettings := CseClusterSettings{ Name: "test-cse", OrganizationId: org.Org.ID, @@ -110,39 +117,9 @@ func (vcd *TestVCD) Test_Cse(check *C) { ServiceCidr: "100.64.0.0/13", AutoRepairOnErrors: true, } - cluster, err := org.CseCreateKubernetesCluster(clusterSettings, 0) check.Assert(err, IsNil) - check.Assert(cluster.CseVersion.String(), Equals, cseVersion.String()) - check.Assert(cluster.Name, Equals, clusterSettings.Name) - check.Assert(cluster.OrganizationId, Equals, clusterSettings.OrganizationId) - check.Assert(cluster.VdcId, Equals, clusterSettings.VdcId) - check.Assert(cluster.NetworkId, Equals, clusterSettings.NetworkId) - check.Assert(cluster.KubernetesTemplateOvaId, Equals, clusterSettings.KubernetesTemplateOvaId) - check.Assert(cluster.ControlPlane, DeepEquals, clusterSettings.ControlPlane) - check.Assert(cluster.WorkerPools, DeepEquals, clusterSettings.WorkerPools) - check.Assert(cluster.DefaultStorageClass, NotNil) - check.Assert(*cluster.DefaultStorageClass, DeepEquals, *clusterSettings.DefaultStorageClass) - check.Assert(cluster.Owner, Equals, clusterSettings.Owner) - check.Assert(cluster.ApiToken, Not(Equals), clusterSettings.ApiToken) - check.Assert(cluster.ApiToken, Equals, "******") // This one can't be recovered - check.Assert(cluster.NodeHealthCheck, Equals, clusterSettings.NodeHealthCheck) - check.Assert(cluster.PodCidr, Equals, clusterSettings.PodCidr) - check.Assert(cluster.ServiceCidr, Equals, clusterSettings.ServiceCidr) - check.Assert(cluster.SshPublicKey, Equals, clusterSettings.SshPublicKey) - check.Assert(cluster.VirtualIpSubnet, Equals, clusterSettings.VirtualIpSubnet) - check.Assert(cluster.AutoRepairOnErrors, Equals, clusterSettings.AutoRepairOnErrors) - check.Assert(cluster.VirtualIpSubnet, Equals, clusterSettings.VirtualIpSubnet) - check.Assert(true, Equals, strings.Contains(cluster.ID, "urn:vcloud:entity:vmware:capvcdCluster:")) - check.Assert(cluster.Etag, Not(Equals), "") - check.Assert(cluster.KubernetesVersion, Equals, tkgBundle.KubernetesVersion) - check.Assert(cluster.TkgVersion, Equals, tkgBundle.TkgVersion) - check.Assert(cluster.CapvcdVersion, Not(Equals), "") - check.Assert(cluster.CpiVersion, Not(Equals), "") - check.Assert(cluster.CsiVersion, Not(Equals), "") - check.Assert(len(cluster.ClusterResourceSetBindings), Not(Equals), 0) - check.Assert(cluster.State, Equals, "provisioned") - check.Assert(len(cluster.Events), Not(Equals), 0) + assertCseClusterCreation(check, cluster, clusterSettings, tkgBundle) kubeconfig, err := cluster.GetKubeconfig() check.Assert(err, IsNil) @@ -156,68 +133,14 @@ func (vcd *TestVCD) Test_Cse(check *C) { clusterGet, err := vcd.client.CseGetKubernetesClusterById(cluster.ID) check.Assert(err, IsNil) - check.Assert(cluster.CseVersion.String(), Equals, clusterGet.CseVersion.String()) - check.Assert(cluster.Name, Equals, clusterGet.Name) - check.Assert(cluster.OrganizationId, Equals, clusterGet.OrganizationId) - check.Assert(cluster.VdcId, Equals, clusterGet.VdcId) - check.Assert(cluster.NetworkId, Equals, clusterGet.NetworkId) - check.Assert(cluster.KubernetesTemplateOvaId, Equals, clusterGet.KubernetesTemplateOvaId) - check.Assert(cluster.ControlPlane, DeepEquals, clusterGet.ControlPlane) - check.Assert(cluster.WorkerPools, DeepEquals, clusterGet.WorkerPools) - check.Assert(cluster.DefaultStorageClass, NotNil) - check.Assert(*cluster.DefaultStorageClass, DeepEquals, *clusterGet.DefaultStorageClass) - check.Assert(cluster.Owner, Equals, clusterGet.Owner) - check.Assert(cluster.ApiToken, Not(Equals), clusterGet.ApiToken) - check.Assert(clusterGet.ApiToken, Equals, "******") // This one can't be recovered - check.Assert(cluster.NodeHealthCheck, Equals, clusterGet.NodeHealthCheck) - check.Assert(cluster.PodCidr, Equals, clusterGet.PodCidr) - check.Assert(cluster.ServiceCidr, Equals, clusterGet.ServiceCidr) - check.Assert(cluster.SshPublicKey, Equals, clusterGet.SshPublicKey) - check.Assert(cluster.VirtualIpSubnet, Equals, clusterGet.VirtualIpSubnet) - check.Assert(cluster.AutoRepairOnErrors, Equals, clusterGet.AutoRepairOnErrors) - check.Assert(cluster.VirtualIpSubnet, Equals, clusterGet.VirtualIpSubnet) - check.Assert(cluster.ID, Equals, clusterGet.ID) + assertCseClusterEquals(check, clusterGet, cluster) check.Assert(clusterGet.Etag, Not(Equals), "") - check.Assert(cluster.KubernetesVersion, Equals, clusterGet.KubernetesVersion) - check.Assert(cluster.TkgVersion.String(), Equals, clusterGet.TkgVersion.String()) - check.Assert(cluster.CapvcdVersion.String(), Equals, clusterGet.CapvcdVersion.String()) - check.Assert(cluster.ClusterResourceSetBindings, DeepEquals, clusterGet.ClusterResourceSetBindings) - check.Assert(cluster.CpiVersion.String(), Equals, clusterGet.CpiVersion.String()) - check.Assert(cluster.CsiVersion.String(), Equals, clusterGet.CsiVersion.String()) - check.Assert(cluster.State, Equals, clusterGet.State) allClusters, err := org.CseGetKubernetesClustersByName(clusterGet.CseVersion, clusterGet.Name) check.Assert(err, IsNil) check.Assert(len(allClusters), Equals, 1) - check.Assert(cluster.CseVersion.String(), Equals, allClusters[0].CseVersion.String()) - check.Assert(cluster.Name, Equals, allClusters[0].Name) - check.Assert(cluster.OrganizationId, Equals, allClusters[0].OrganizationId) - check.Assert(cluster.VdcId, Equals, allClusters[0].VdcId) - check.Assert(cluster.NetworkId, Equals, allClusters[0].NetworkId) - check.Assert(cluster.KubernetesTemplateOvaId, Equals, allClusters[0].KubernetesTemplateOvaId) - check.Assert(cluster.ControlPlane, DeepEquals, allClusters[0].ControlPlane) - check.Assert(cluster.WorkerPools, DeepEquals, allClusters[0].WorkerPools) - check.Assert(cluster.DefaultStorageClass, NotNil) - check.Assert(*cluster.DefaultStorageClass, DeepEquals, *allClusters[0].DefaultStorageClass) - check.Assert(cluster.Owner, Equals, allClusters[0].Owner) - check.Assert(cluster.ApiToken, Not(Equals), allClusters[0].ApiToken) - check.Assert(allClusters[0].ApiToken, Equals, "******") // This one can't be recovered - check.Assert(cluster.NodeHealthCheck, Equals, allClusters[0].NodeHealthCheck) - check.Assert(cluster.PodCidr, Equals, allClusters[0].PodCidr) - check.Assert(cluster.ServiceCidr, Equals, allClusters[0].ServiceCidr) - check.Assert(cluster.SshPublicKey, Equals, allClusters[0].SshPublicKey) - check.Assert(cluster.VirtualIpSubnet, Equals, allClusters[0].VirtualIpSubnet) - check.Assert(cluster.AutoRepairOnErrors, Equals, allClusters[0].AutoRepairOnErrors) - check.Assert(cluster.VirtualIpSubnet, Equals, allClusters[0].VirtualIpSubnet) - check.Assert(cluster.ID, Equals, allClusters[0].ID) - check.Assert(allClusters[0].Etag, Not(Equals), "") - check.Assert(cluster.KubernetesVersion, Equals, allClusters[0].KubernetesVersion) - check.Assert(cluster.TkgVersion.String(), Equals, allClusters[0].TkgVersion.String()) - check.Assert(cluster.CapvcdVersion.String(), Equals, allClusters[0].CapvcdVersion.String()) - check.Assert(cluster.ClusterResourceSetBindings, DeepEquals, allClusters[0].ClusterResourceSetBindings) - check.Assert(cluster.CpiVersion.String(), Equals, allClusters[0].CpiVersion.String()) - check.Assert(cluster.CsiVersion.String(), Equals, allClusters[0].CsiVersion.String()) - check.Assert(cluster.State, Equals, allClusters[0].State) + assertCseClusterEquals(check, allClusters[0], clusterGet) + check.Assert(allClusters[0].Etag, Equals, "") // Can't recover ETag by name // Update worker pool from 1 node to 2 // Pre-check. This should be 1, as it was created with just 1 pool @@ -240,14 +163,181 @@ func (vcd *TestVCD) Test_Cse(check *C) { } check.Assert(foundWorkerPool, Equals, true) - err = cluster.Delete(0) + // Perform the update + err = cluster.AddWorkerPools([]CseWorkerPoolSettings{{ + Name: "new-pool", + MachineCount: 1, + DiskSizeGi: 20, + }}, true) check.Assert(err, IsNil) -} -func (vcd *TestVCD) Test_Deleteme(check *C) { - cluster, err := vcd.client.CseGetKubernetesClusterById("urn:vcloud:entity:vmware:capvcdCluster:bac3333c-51ca-4b3e-a239-ae8669f899e6") + // Post-check. This should be 2, as it should have scaled up + foundWorkerPool = false + for _, nodePool := range cluster.WorkerPools { + if nodePool.Name == "new-pool" { + foundWorkerPool = true + check.Assert(nodePool.MachineCount, Equals, 1) + check.Assert(nodePool.DiskSizeGi, Equals, 20) + check.Assert(nodePool.SizingPolicyId, Equals, "") + check.Assert(nodePool.StorageProfileId, Equals, "") + } + } + check.Assert(foundWorkerPool, Equals, true) + + // Update control plane from 1 node to 2 + // Pre-check. This should be 1, as it was created with just 1 pool + check.Assert(cluster.ControlPlane.MachineCount, Equals, 1) + + // Perform the update + err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 2}, true) check.Assert(err, IsNil) + // Post-check. This should be 2, as it should have scaled up + check.Assert(cluster.ControlPlane.MachineCount, Equals, 2) + + // Update the node health check. + // Pre-check. This should be true, as it was created + check.Assert(cluster.NodeHealthCheck, Equals, true) + // Perform the update err = cluster.SetNodeHealthCheck(false, true) check.Assert(err, IsNil) + // Post-check. This should be false now + check.Assert(cluster.NodeHealthCheck, Equals, false) + + // Update the auto repair flag. + err = cluster.SetAutoRepairOnErrors(false, true) + if cluster.CseVersion.GreaterThanOrEqual(v411) { + check.Assert(err, IsNil) + check.Assert(cluster.NodeHealthCheck, Equals, false) + } else { + check.Assert(err, IsNil) + check.Assert(cluster.NodeHealthCheck, Equals, false) + } + + upgradeOvas, err := cluster.GetSupportedUpgrades(true) + check.Assert(err, IsNil) + if len(upgradeOvas) > 0 { + err = cluster.UpgradeCluster(upgradeOvas[0].ID, true) + check.Assert(err, IsNil) + check.Assert(cluster.KubernetesVersion, Not(Equals), clusterGet.KubernetesVersion) + check.Assert(cluster.TkgVersion, Not(Equals), clusterGet.TkgVersion) + check.Assert(cluster.KubernetesTemplateOvaId, Not(Equals), clusterGet.KubernetesTemplateOvaId) + upgradeOvas, err = cluster.GetSupportedUpgrades(true) + check.Assert(err, IsNil) + check.Assert(len(upgradeOvas), Equals, 0) + } else { + fmt.Println("WARNING: CseKubernetesCluster.UpgradeCluster method not tested. It was skipped as there's no OVA to upgrade the cluster") + } + + // Helps to delete the cluster faster, also tests generic update method + err = cluster.Update(CseClusterUpdateInput{ + ControlPlane: &CseControlPlaneUpdateInput{MachineCount: 1}, + WorkerPools: &map[string]CseWorkerPoolUpdateInput{ + clusterSettings.WorkerPools[0].Name: { + MachineCount: 0, + }, + "new-pool": { + MachineCount: 0, + }, + }, + }, true) + check.Assert(err, IsNil) + check.Assert(cluster.ControlPlane.MachineCount, Equals, 1) + check.Assert(cluster.WorkerPools[0].MachineCount, Equals, 0) + check.Assert(cluster.WorkerPools[1].MachineCount, Equals, 0) + + err = cluster.Delete(0) + check.Assert(err, IsNil) +} + +func assertCseClusterCreation(check *C, createdCluster *CseKubernetesCluster, settings CseClusterSettings, expectedKubernetesData tkgVersionBundle) { + check.Assert(createdCluster, NotNil) + check.Assert(createdCluster.CseVersion.Original(), Equals, settings.CseVersion.Original()) + check.Assert(createdCluster.Name, Equals, settings.Name) + check.Assert(createdCluster.OrganizationId, Equals, settings.OrganizationId) + check.Assert(createdCluster.VdcId, Equals, settings.VdcId) + check.Assert(createdCluster.NetworkId, Equals, settings.NetworkId) + check.Assert(createdCluster.KubernetesTemplateOvaId, Equals, settings.KubernetesTemplateOvaId) + check.Assert(createdCluster.ControlPlane.MachineCount, Equals, settings.ControlPlane.MachineCount) + check.Assert(createdCluster.ControlPlane.SizingPolicyId, Equals, settings.ControlPlane.SizingPolicyId) + check.Assert(createdCluster.ControlPlane.PlacementPolicyId, Equals, settings.ControlPlane.PlacementPolicyId) + check.Assert(createdCluster.ControlPlane.StorageProfileId, Equals, settings.ControlPlane.StorageProfileId) + check.Assert(createdCluster.ControlPlane.DiskSizeGi, Equals, settings.ControlPlane.DiskSizeGi) + if settings.ControlPlane.Ip != "" { + check.Assert(createdCluster.ControlPlane.Ip, Equals, settings.ControlPlane.Ip) + } else { + check.Assert(createdCluster.ControlPlane.Ip, Not(Equals), "") + } + check.Assert(createdCluster.WorkerPools, DeepEquals, settings.WorkerPools) + if settings.DefaultStorageClass != nil { + check.Assert(createdCluster.DefaultStorageClass, NotNil) + check.Assert(*createdCluster.DefaultStorageClass, DeepEquals, *settings.DefaultStorageClass) + } + if settings.Owner != "" { + check.Assert(createdCluster.Owner, Equals, settings.Owner) + } else { + check.Assert(createdCluster.Owner, Not(Equals), "") + } + check.Assert(createdCluster.ApiToken, Not(Equals), settings.ApiToken) + check.Assert(createdCluster.ApiToken, Equals, "******") // This one can't be recovered + check.Assert(createdCluster.NodeHealthCheck, Equals, settings.NodeHealthCheck) + check.Assert(createdCluster.PodCidr, Equals, settings.PodCidr) + check.Assert(createdCluster.ServiceCidr, Equals, settings.ServiceCidr) + check.Assert(createdCluster.SshPublicKey, Equals, settings.SshPublicKey) + check.Assert(createdCluster.VirtualIpSubnet, Equals, settings.VirtualIpSubnet) + + v411, err := semver.NewVersion("4.1.1") + check.Assert(err, IsNil) + if settings.CseVersion.GreaterThanOrEqual(v411) { + // Since CSE 4.1.1, the flag is automatically switched off when the cluster is created + check.Assert(createdCluster.AutoRepairOnErrors, Equals, false) + } else { + check.Assert(createdCluster.AutoRepairOnErrors, Equals, settings.AutoRepairOnErrors) + } + check.Assert(createdCluster.VirtualIpSubnet, Equals, settings.VirtualIpSubnet) + check.Assert(true, Equals, strings.Contains(createdCluster.ID, "urn:vcloud:entity:vmware:capvcdCluster:")) + check.Assert(createdCluster.Etag, Not(Equals), "") + check.Assert(createdCluster.KubernetesVersion.Original(), Equals, expectedKubernetesData.KubernetesVersion) + check.Assert(createdCluster.TkgVersion.Original(), Equals, expectedKubernetesData.TkgVersion) + check.Assert(createdCluster.CapvcdVersion.Original(), Not(Equals), "") + check.Assert(createdCluster.CpiVersion.Original(), Not(Equals), "") + check.Assert(createdCluster.CsiVersion.Original(), Not(Equals), "") + check.Assert(len(createdCluster.ClusterResourceSetBindings), Not(Equals), 0) + check.Assert(createdCluster.State, Equals, "provisioned") + check.Assert(len(createdCluster.Events), Not(Equals), 0) +} + +func assertCseClusterEquals(check *C, obtainedCluster, expectedCluster *CseKubernetesCluster) { + check.Assert(expectedCluster, NotNil) + check.Assert(obtainedCluster, NotNil) + check.Assert(obtainedCluster.CseVersion.Original(), Equals, expectedCluster.CseVersion.Original()) + check.Assert(obtainedCluster.Name, Equals, expectedCluster.Name) + check.Assert(obtainedCluster.OrganizationId, Equals, expectedCluster.OrganizationId) + check.Assert(obtainedCluster.VdcId, Equals, expectedCluster.VdcId) + check.Assert(obtainedCluster.NetworkId, Equals, expectedCluster.NetworkId) + check.Assert(obtainedCluster.KubernetesTemplateOvaId, Equals, expectedCluster.KubernetesTemplateOvaId) + check.Assert(obtainedCluster.ControlPlane, DeepEquals, expectedCluster.ControlPlane) + check.Assert(obtainedCluster.WorkerPools, DeepEquals, expectedCluster.WorkerPools) + if expectedCluster.DefaultStorageClass != nil { + check.Assert(obtainedCluster.DefaultStorageClass, NotNil) + check.Assert(*obtainedCluster.DefaultStorageClass, DeepEquals, *expectedCluster.DefaultStorageClass) + } + check.Assert(obtainedCluster.Owner, Equals, expectedCluster.Owner) + check.Assert(obtainedCluster.ApiToken, Equals, "******") // This one can't be recovered + check.Assert(obtainedCluster.NodeHealthCheck, Equals, expectedCluster.NodeHealthCheck) + check.Assert(obtainedCluster.PodCidr, Equals, expectedCluster.PodCidr) + check.Assert(obtainedCluster.ServiceCidr, Equals, expectedCluster.ServiceCidr) + check.Assert(obtainedCluster.SshPublicKey, Equals, expectedCluster.SshPublicKey) + check.Assert(obtainedCluster.VirtualIpSubnet, Equals, expectedCluster.VirtualIpSubnet) + check.Assert(obtainedCluster.AutoRepairOnErrors, Equals, expectedCluster.AutoRepairOnErrors) + check.Assert(obtainedCluster.VirtualIpSubnet, Equals, expectedCluster.VirtualIpSubnet) + check.Assert(obtainedCluster.ID, Equals, expectedCluster.ID) + check.Assert(obtainedCluster.KubernetesVersion.Original(), Equals, expectedCluster.KubernetesVersion.Original()) + check.Assert(obtainedCluster.TkgVersion.Original(), Equals, expectedCluster.TkgVersion.Original()) + check.Assert(obtainedCluster.CapvcdVersion.Original(), Equals, expectedCluster.CapvcdVersion.Original()) + check.Assert(obtainedCluster.CpiVersion.Original(), Equals, expectedCluster.CpiVersion.Original()) + check.Assert(obtainedCluster.CsiVersion.Original(), Equals, expectedCluster.CsiVersion.Original()) + check.Assert(obtainedCluster.ClusterResourceSetBindings, DeepEquals, expectedCluster.ClusterResourceSetBindings) + check.Assert(obtainedCluster.State, Equals, expectedCluster.State) + check.Assert(len(obtainedCluster.Events) >= len(expectedCluster.Events), Equals, true) } diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 06cad592f..fcb7e87f1 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -157,8 +157,8 @@ func cseUpdateKubernetesTemplateInYaml(yamlDocuments []map[string]interface{}, k // cseUpdateControlPlaneInYaml modifies the given Kubernetes cluster YAML contents by changing the Control Plane with the input parameters. func cseUpdateControlPlaneInYaml(yamlDocuments []map[string]interface{}, input CseControlPlaneUpdateInput) error { - if input.MachineCount < 0 { - return fmt.Errorf("incorrect machine count for Control Plane: %d. Should be at least 0", input.MachineCount) + if input.MachineCount < 1 || input.MachineCount%2 == 0 { + return fmt.Errorf("incorrect machine count for Control Plane: %d. Should be at least 1 and an odd number", input.MachineCount) } updated := false diff --git a/govcd/sample_govcd_test_config.yaml b/govcd/sample_govcd_test_config.yaml index 10f923ac1..f54375db4 100644 --- a/govcd/sample_govcd_test_config.yaml +++ b/govcd/sample_govcd_test_config.yaml @@ -254,6 +254,8 @@ media: # A valid UI Plugin to use in tests uiPluginPath: ../test-resources/ui_plugin.zip cse: + # The CSE version installed in VCD + version: "4.2.0" # The organization where Container Service Extension (CSE) Server is running solutionsOrg: "solutions_org" # The organization where the Kubernetes clusters are created From a0f2b676b0e15dff6e2c9dbce624b020c708c8dc Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 12:09:02 +0100 Subject: [PATCH 073/115] Fix tests Signed-off-by: abarreiro --- govcd/cse_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 04463c4e0..e5366c180 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -234,7 +234,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { ControlPlane: &CseControlPlaneUpdateInput{MachineCount: 1}, WorkerPools: &map[string]CseWorkerPoolUpdateInput{ clusterSettings.WorkerPools[0].Name: { - MachineCount: 0, + MachineCount: 1, }, "new-pool": { MachineCount: 0, @@ -243,7 +243,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { }, true) check.Assert(err, IsNil) check.Assert(cluster.ControlPlane.MachineCount, Equals, 1) - check.Assert(cluster.WorkerPools[0].MachineCount, Equals, 0) + check.Assert(cluster.WorkerPools[0].MachineCount, Equals, 1) check.Assert(cluster.WorkerPools[1].MachineCount, Equals, 0) err = cluster.Delete(0) From 39648045ac28090aae517152e782d004b203e8de Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 12:16:20 +0100 Subject: [PATCH 074/115] Fix test Signed-off-by: abarreiro --- govcd/cse_test.go | 41 +++++++++-------------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index e5366c180..fda7feccb 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -36,7 +36,6 @@ func (vcd *TestVCD) Test_Cse(check *C) { } // Prerequisites: We need to read several items before creating the cluster. - org, err := vcd.client.GetOrgByName(vcd.config.Cse.TenantOrg) check.Assert(err, IsNil) @@ -79,9 +78,6 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(err, IsNil) check.Assert(cseVersion, NotNil) - v411, err := semver.NewVersion("4.1.1") - check.Assert(err, IsNil) - // Create the cluster clusterSettings := CseClusterSettings{ Name: "test-cse", @@ -143,17 +139,8 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(allClusters[0].Etag, Equals, "") // Can't recover ETag by name // Update worker pool from 1 node to 2 - // Pre-check. This should be 1, as it was created with just 1 pool - for _, nodePool := range cluster.WorkerPools { - if nodePool.Name == clusterSettings.WorkerPools[0].Name { - check.Assert(nodePool.MachineCount, Equals, 1) - } - } - // Perform the update err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{clusterSettings.WorkerPools[0].Name: {MachineCount: 2}}, true) check.Assert(err, IsNil) - - // Post-check. This should be 2, as it should have scaled up foundWorkerPool := false for _, nodePool := range cluster.WorkerPools { if nodePool.Name == clusterSettings.WorkerPools[0].Name { @@ -163,15 +150,13 @@ func (vcd *TestVCD) Test_Cse(check *C) { } check.Assert(foundWorkerPool, Equals, true) - // Perform the update + // Add a new worker pool err = cluster.AddWorkerPools([]CseWorkerPoolSettings{{ Name: "new-pool", MachineCount: 1, DiskSizeGi: 20, }}, true) check.Assert(err, IsNil) - - // Post-check. This should be 2, as it should have scaled up foundWorkerPool = false for _, nodePool := range cluster.WorkerPools { if nodePool.Name == "new-pool" { @@ -185,35 +170,27 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(foundWorkerPool, Equals, true) // Update control plane from 1 node to 2 - // Pre-check. This should be 1, as it was created with just 1 pool - check.Assert(cluster.ControlPlane.MachineCount, Equals, 1) - - // Perform the update err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 2}, true) check.Assert(err, IsNil) - - // Post-check. This should be 2, as it should have scaled up check.Assert(cluster.ControlPlane.MachineCount, Equals, 2) - // Update the node health check. - // Pre-check. This should be true, as it was created - check.Assert(cluster.NodeHealthCheck, Equals, true) - // Perform the update + // Turn off the node health check err = cluster.SetNodeHealthCheck(false, true) check.Assert(err, IsNil) - // Post-check. This should be false now check.Assert(cluster.NodeHealthCheck, Equals, false) - // Update the auto repair flag. + // Update the auto repair flag + v411, err := semver.NewVersion("4.1.1") + check.Assert(err, IsNil) err = cluster.SetAutoRepairOnErrors(false, true) - if cluster.CseVersion.GreaterThanOrEqual(v411) { - check.Assert(err, IsNil) - check.Assert(cluster.NodeHealthCheck, Equals, false) + if cluster.CseVersion.GreaterThan(v411) { + check.Assert(err, NotNil) // Can't be changed since CSE 4.1.1 } else { check.Assert(err, IsNil) - check.Assert(cluster.NodeHealthCheck, Equals, false) } + check.Assert(cluster.NodeHealthCheck, Equals, false) + // Upgrade the cluster if possible upgradeOvas, err := cluster.GetSupportedUpgrades(true) check.Assert(err, IsNil) if len(upgradeOvas) > 0 { From 8f06a6762d3cf992518582053e54381bb780b7f7 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 12:43:36 +0100 Subject: [PATCH 075/115] Improve test checks Signed-off-by: abarreiro --- govcd/cse_test.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index fda7feccb..de40c842d 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -13,27 +13,28 @@ import ( . "gopkg.in/check.v1" "net/url" "os" + "reflect" "strings" ) -const ( - TestRequiresCseConfiguration = "Test %s requires CSE configuration details" -) - -func skipCseTests(testConfig TestConfig) bool { +func requireCseConfig(check *C, testConfig TestConfig) { + skippedPrefix := fmt.Sprintf("skipped %s because:", check.TestName()) if cse := os.Getenv("TEST_VCD_CSE"); cse == "" { - return true + check.Skip(fmt.Sprintf("%s the environment variable TEST_VCD_CSE is not set", skippedPrefix)) } - return testConfig.Cse.Version == "" || testConfig.Cse.SolutionsOrg == "" || testConfig.Cse.TenantOrg == "" || testConfig.Cse.OvaName == "" || - testConfig.Cse.RoutedNetwork == "" || testConfig.Cse.EdgeGateway == "" || testConfig.Cse.OvaCatalog == "" || testConfig.Cse.TenantVdc == "" || - testConfig.Cse.StorageProfile == "" + cseConfigValues := reflect.ValueOf(testConfig.Cse) + cseConfigType := cseConfigValues.Type() + for i := 0; i < cseConfigValues.NumField(); i++ { + if cseConfigValues.Field(i).String() == "" { + check.Skip(fmt.Sprintf("%s the config value '%s' inside 'cse' block of govcd_test_config.yaml is not set", skippedPrefix, strings.ToLower(cseConfigType.Field(i).Name))) + } + } + check.Skip("foo") } // Test_Cse tests all possible combinations of the CSE CRUD operations. func (vcd *TestVCD) Test_Cse(check *C) { - if skipCseTests(vcd.config) { - check.Skip(fmt.Sprintf(TestRequiresCseConfiguration, check.TestName())) - } + requireCseConfig(check, vcd.config) // Prerequisites: We need to read several items before creating the cluster. org, err := vcd.client.GetOrgByName(vcd.config.Cse.TenantOrg) From b716d7c31d0b3d6120a417d58557e786eb38a9b5 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 12:45:39 +0100 Subject: [PATCH 076/115] Improve test checks Signed-off-by: abarreiro --- govcd/cse_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index de40c842d..afd108b5e 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -29,7 +29,6 @@ func requireCseConfig(check *C, testConfig TestConfig) { check.Skip(fmt.Sprintf("%s the config value '%s' inside 'cse' block of govcd_test_config.yaml is not set", skippedPrefix, strings.ToLower(cseConfigType.Field(i).Name))) } } - check.Skip("foo") } // Test_Cse tests all possible combinations of the CSE CRUD operations. From f49d3418d89c12553b645009658dac97896ff179 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 15:27:55 +0100 Subject: [PATCH 077/115] Improve RDE creation Signed-off-by: abarreiro --- govcd/api_vcd_test.go | 3 -- govcd/defined_entity.go | 66 ++++++++++++++++++++--------------------- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/govcd/api_vcd_test.go b/govcd/api_vcd_test.go index 6d8cec6d4..16bc66855 100644 --- a/govcd/api_vcd_test.go +++ b/govcd/api_vcd_test.go @@ -250,9 +250,6 @@ type TestConfig struct { OvaCatalog string `yaml:"ovaCatalog,omitempty"` OvaName string `yaml:"ovaName,omitempty"` } `yaml:"cse,omitempty"` - Dse struct { - IsoName string `yaml:"isoName,omitempty"` - } `yaml:"dse,omitempty"` } // Test struct for vcloud-director. diff --git a/govcd/defined_entity.go b/govcd/defined_entity.go index b7cba4c61..ef9fe9a31 100644 --- a/govcd/defined_entity.go +++ b/govcd/defined_entity.go @@ -7,10 +7,9 @@ package govcd import ( "encoding/json" "fmt" - "net/url" - "time" - "github.com/vmware/go-vcloud-director/v2/types/v56" + "net/url" + "strings" ) const ( @@ -372,11 +371,11 @@ func getRdeById(client *Client, id string) (*DefinedEntity, error) { // and the generated VCD task will remain at 1% until resolved. func (rdeType *DefinedEntityType) CreateRde(entity types.DefinedEntity, tenantContext *TenantContext) (*DefinedEntity, error) { entity.EntityType = rdeType.DefinedEntityType.ID - err := createRde(rdeType.client, entity, tenantContext) + task, err := createRde(rdeType.client, entity, tenantContext) if err != nil { return nil, err } - return pollPreCreatedRde(rdeType.client, rdeType.DefinedEntityType.Vendor, rdeType.DefinedEntityType.Nss, rdeType.DefinedEntityType.Version, entity.Name, 5) + return pollCreatedRdeTask(rdeType.client, task) } // CreateRde creates an entity of the type of the given vendor, nss and version. @@ -391,11 +390,11 @@ func (vcdClient *VCDClient) CreateRde(vendor, nss, version string, entity types. // and the generated VCD task will remain at 1% until resolved. func createRdeAndPoll(client *Client, vendor, nss, version string, entity types.DefinedEntity, tenantContext *TenantContext) (*DefinedEntity, error) { entity.EntityType = fmt.Sprintf("urn:vcloud:type:%s:%s:%s", vendor, nss, version) - err := createRde(client, entity, tenantContext) + task, err := createRde(client, entity, tenantContext) if err != nil { return nil, err } - return pollPreCreatedRde(client, vendor, nss, version, entity.Name, 5) + return pollCreatedRdeTask(client, task) } // CreateRde creates an entity of the type of the receiver Runtime Defined Entity (RDE) type. @@ -403,54 +402,53 @@ func createRdeAndPoll(client *Client, vendor, nss, version string, entity types. // it must match the type ID of the receiver RDE type. // NOTE: After RDE creation, some actor should Resolve it, otherwise the RDE state will be "PRE_CREATED" // and the generated VCD task will remain at 1% until resolved. -func createRde(client *Client, entity types.DefinedEntity, tenantContext *TenantContext) error { +func createRde(client *Client, entity types.DefinedEntity, tenantContext *TenantContext) (*Task, error) { if entity.EntityType == "" { - return fmt.Errorf("ID of the Runtime Defined Entity type is empty") + return nil, fmt.Errorf("ID of the Runtime Defined Entity type is empty") } if entity.Entity == nil || len(entity.Entity) == 0 { - return fmt.Errorf("the entity JSON is empty") + return nil, fmt.Errorf("the entity JSON is empty") } endpoint := types.OpenApiPathVersion1_0_0 + types.OpenApiEndpointRdeEntityTypes apiVersion, err := client.getOpenApiHighestElevatedVersion(endpoint) if err != nil { - return err + return nil, err } urlRef, err := client.OpenApiBuildEndpoint(endpoint, entity.EntityType) if err != nil { - return err + return nil, err } - _, err = client.OpenApiPostItemAsyncWithHeaders(apiVersion, urlRef, nil, entity, getTenantContextHeader(tenantContext)) + task, err := client.OpenApiPutItemAsync(apiVersion, urlRef, nil, entity, getTenantContextHeader(tenantContext)) if err != nil { - return err + return nil, err } - return nil + return &task, nil } -// pollPreCreatedRde polls VCD for a given amount of tries, to search for the RDE in state PRE_CREATED -// that corresponds to the given vendor, nss, version and name. -// This function can be useful on RDE creation, as VCD just returns a task that remains at 1% until the RDE is resolved, -// hence one needs to re-fetch the recently created RDE manually. -func pollPreCreatedRde(client *Client, vendor, nss, version, name string, tries int) (*DefinedEntity, error) { - var rdes []*DefinedEntity - var err error - for i := 0; i < tries; i++ { - rdes, err = getRdesByName(client, vendor, nss, version, name) - if err == nil { - for _, rde := range rdes { - // This doesn't really guarantee that the chosen RDE is the one we want, but there's no other way of - // fine-graining - if rde.DefinedEntity.State != nil && *rde.DefinedEntity.State == "PRE_CREATED" { - return rde, nil - } - } +// pollCreatedRdeTask polls VCD for a given amount of tries, to search for the RDE in the given Task that should have +// been created as result of creating that RDE. +func pollCreatedRdeTask(client *Client, task *Task) (*DefinedEntity, error) { + if task.Task == nil { + return nil, fmt.Errorf("could not retrieve the RDE from task, as it is nil") + } + rdeId := "" + if task.Task.Owner == nil { + // Try to retrieve the ID from the "Operation" field + beginning := strings.LastIndex(task.Task.Operation, "(") + end := strings.LastIndex(task.Task.Operation, ")") + if beginning < 0 || end < 0 || beginning >= end { + return nil, fmt.Errorf("could not retrieve the RDE from the task with ID '%s'", task.Task.ID) } - time.Sleep(3 * time.Second) + rdeId = task.Task.Operation[beginning+1 : end] + } else { + rdeId = task.Task.Owner.ID } - return nil, fmt.Errorf("could not create RDE, failed during retrieval after creation: %s", err) + + return getRdeById(client, rdeId) } // Resolve needs to be called after an RDE is successfully created. It makes the receiver RDE usable if the JSON entity From 3bd2203c1128095970499a17f4e3dbbcec94380f Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 15:28:55 +0100 Subject: [PATCH 078/115] Improve RDE creation Signed-off-by: abarreiro --- govcd/cse.go | 2 +- govcd/defined_entity.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 56b6fd651..b4da61ac0 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -56,7 +56,7 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) return "", err } - rde, err := createRdeAndPoll(org.client, "vmware", "capvcdCluster", cseSubcomponents.CapvcdRdeTypeVersion, + rde, err := createRdeAndGetFromTask(org.client, "vmware", "capvcdCluster", cseSubcomponents.CapvcdRdeTypeVersion, types.DefinedEntity{ EntityType: internalSettings.RdeType.ID, Name: internalSettings.Name, diff --git a/govcd/defined_entity.go b/govcd/defined_entity.go index ef9fe9a31..f1797e24c 100644 --- a/govcd/defined_entity.go +++ b/govcd/defined_entity.go @@ -375,26 +375,26 @@ func (rdeType *DefinedEntityType) CreateRde(entity types.DefinedEntity, tenantCo if err != nil { return nil, err } - return pollCreatedRdeTask(rdeType.client, task) + return getRdeFromTask(rdeType.client, task) } // CreateRde creates an entity of the type of the given vendor, nss and version. // NOTE: After RDE creation, some actor should Resolve it, otherwise the RDE state will be "PRE_CREATED" // and the generated VCD task will remain at 1% until resolved. func (vcdClient *VCDClient) CreateRde(vendor, nss, version string, entity types.DefinedEntity, tenantContext *TenantContext) (*DefinedEntity, error) { - return createRdeAndPoll(&vcdClient.Client, vendor, nss, version, entity, tenantContext) + return createRdeAndGetFromTask(&vcdClient.Client, vendor, nss, version, entity, tenantContext) } -// createRdeAndPoll creates an entity of the type of the given vendor, nss and version. +// createRdeAndGetFromTask creates an entity of the type of the given vendor, nss and version. // NOTE: After RDE creation, some actor should Resolve it, otherwise the RDE state will be "PRE_CREATED" // and the generated VCD task will remain at 1% until resolved. -func createRdeAndPoll(client *Client, vendor, nss, version string, entity types.DefinedEntity, tenantContext *TenantContext) (*DefinedEntity, error) { +func createRdeAndGetFromTask(client *Client, vendor, nss, version string, entity types.DefinedEntity, tenantContext *TenantContext) (*DefinedEntity, error) { entity.EntityType = fmt.Sprintf("urn:vcloud:type:%s:%s:%s", vendor, nss, version) task, err := createRde(client, entity, tenantContext) if err != nil { return nil, err } - return pollCreatedRdeTask(client, task) + return getRdeFromTask(client, task) } // CreateRde creates an entity of the type of the receiver Runtime Defined Entity (RDE) type. @@ -429,9 +429,9 @@ func createRde(client *Client, entity types.DefinedEntity, tenantContext *Tenant return &task, nil } -// pollCreatedRdeTask polls VCD for a given amount of tries, to search for the RDE in the given Task that should have +// getRdeFromTask polls VCD for a given amount of tries, to search for the RDE in the given Task that should have // been created as result of creating that RDE. -func pollCreatedRdeTask(client *Client, task *Task) (*DefinedEntity, error) { +func getRdeFromTask(client *Client, task *Task) (*DefinedEntity, error) { if task.Task == nil { return nil, fmt.Errorf("could not retrieve the RDE from task, as it is nil") } From da27836af67d9cde9bebbdc7cbebaa9312a50c3a Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 15:45:06 +0100 Subject: [PATCH 079/115] Self-review Signed-off-by: abarreiro --- govcd/cse.go | 23 ++++++++++++++++------- govcd/cse_test.go | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index b4da61ac0..5edce13da 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -77,7 +77,7 @@ func (vcdClient *VCDClient) CseGetKubernetesClusterById(id string) (*CseKubernet return getCseKubernetesClusterById(&vcdClient.Client, id) } -// CseGetKubernetesClustersByName retrieves the CSE Kubernetes cluster from VCD with the given name. +// CseGetKubernetesClustersByName retrieves all the CSE Kubernetes clusters from VCD with the given name that belong to the receiver Organization. // Note: The clusters retrieved won't have a valid ETag to perform operations on them. Use VCDClient.CseGetKubernetesClusterById for that instead. func (org *Org) CseGetKubernetesClustersByName(cseVersion semver.Version, name string) ([]*CseKubernetesCluster, error) { cseSubcomponents, err := getCseComponentsVersions(cseVersion) @@ -111,7 +111,7 @@ func getCseKubernetesClusterById(client *Client, clusterId string) (*CseKubernet return cseConvertToCseKubernetesClusterType(rde) } -// Refresh gets the latest information about the receiver cluster and updates its properties. +// Refresh gets the latest information about the receiver CSE Kubernetes cluster and updates its properties. // All cached fields such as the supported OVAs list (from CseKubernetesCluster.GetSupportedUpgrades) are also cleared. func (cluster *CseKubernetesCluster) Refresh() error { refreshed, err := getCseKubernetesClusterById(cluster.client, cluster.ID) @@ -122,8 +122,16 @@ func (cluster *CseKubernetesCluster) Refresh() error { return nil } -// GetKubeconfig retrieves the Kubeconfig from an available cluster. -func (cluster *CseKubernetesCluster) GetKubeconfig() (string, error) { +// GetKubeconfig retrieves the Kubeconfig from an existing CSE Kubernetes cluster that is in provisioned state. +// If refresh=true, it retrieves the latest state of the cluster from VCD before requesting the Kubeconfig. +func (cluster *CseKubernetesCluster) GetKubeconfig(refresh bool) (string, error) { + if refresh { + err := cluster.Refresh() + if err != nil { + return "", err + } + } + if cluster.State != "provisioned" { return "", fmt.Errorf("cannot get a Kubeconfig of a Kubernetes cluster that is not in 'provisioned' state") } @@ -159,6 +167,7 @@ func (cluster *CseKubernetesCluster) GetKubeconfig() (string, error) { // UpdateWorkerPools executes an update on the receiver cluster to change the existing worker pools. // If refresh=true, it retrieves the latest state of the cluster from VCD before updating. +// WARNING: At least one worker pool must have one or more nodes running, otherwise the cluster will be left in an unusable state. func (cluster *CseKubernetesCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ WorkerPools: &input, @@ -235,15 +244,15 @@ func (cluster *CseKubernetesCluster) SetNodeHealthCheck(healthCheckEnabled bool, // SetAutoRepairOnErrors executes an update on the receiver cluster to change the flag that controls the auto-repair // capabilities of CSE. If refresh=true, it retrieves the latest state of the cluster from VCD before updating. -// NOTE: This can only be used in CSE versions < 4.1.1 +// NOTE: This method can only be used in CSE versions < 4.1.1 func (cluster *CseKubernetesCluster) SetAutoRepairOnErrors(autoRepairOnErrors bool, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ AutoRepairOnErrors: &autoRepairOnErrors, }, refresh) } -// Update executes an update on the receiver CSE Kubernetes Cluster on any of the allowed parameters defined in the input type. If refresh=true, -// it retrieves the latest state of the cluster from VCD before updating. +// Update executes an update on the receiver CSE Kubernetes Cluster on any of the allowed parameters defined in the input type. +// If refresh=true, it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh bool) error { if refresh { err := cluster.Refresh() diff --git a/govcd/cse_test.go b/govcd/cse_test.go index afd108b5e..e9c7fef56 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -117,7 +117,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(err, IsNil) assertCseClusterCreation(check, cluster, clusterSettings, tkgBundle) - kubeconfig, err := cluster.GetKubeconfig() + kubeconfig, err := cluster.GetKubeconfig(false) check.Assert(err, IsNil) check.Assert(true, Equals, strings.Contains(kubeconfig, cluster.Name)) check.Assert(true, Equals, strings.Contains(kubeconfig, "client-certificate-data")) From 53740a0126baff0c4fe193d42912ef4875899040 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 15:46:37 +0100 Subject: [PATCH 080/115] Self-review Signed-off-by: abarreiro --- govcd/cse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/govcd/cse.go b/govcd/cse.go index 5edce13da..589063955 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -35,7 +35,7 @@ func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterSettings, timeo // CseCreateKubernetesClusterAsync creates a Kubernetes cluster with the data given as input (CseClusterSettings), but does not // wait for the creation process to finish, so it doesn't monitor for any errors during the process. It returns just the ID of -// the created cluster. One can manually check the status of the cluster with Org.CseGetKubernetesClusterById and the result of this method. +// the created cluster. One can manually check the status of the cluster with VCDClient.CseGetKubernetesClusterById and the result of this method. func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) (string, error) { if org == nil { return "", fmt.Errorf("CseCreateKubernetesClusterAsync cannot be called on a nil Organization receiver") From 38da9d1cb5340f6f28acae2221b6db1428b664de Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 15:53:05 +0100 Subject: [PATCH 081/115] self-review Signed-off-by: abarreiro --- govcd/cse.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 589063955..c390e0afa 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -36,22 +36,22 @@ func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterSettings, timeo // CseCreateKubernetesClusterAsync creates a Kubernetes cluster with the data given as input (CseClusterSettings), but does not // wait for the creation process to finish, so it doesn't monitor for any errors during the process. It returns just the ID of // the created cluster. One can manually check the status of the cluster with VCDClient.CseGetKubernetesClusterById and the result of this method. -func (org *Org) CseCreateKubernetesClusterAsync(clusterData CseClusterSettings) (string, error) { +func (org *Org) CseCreateKubernetesClusterAsync(clusterSettings CseClusterSettings) (string, error) { if org == nil { return "", fmt.Errorf("CseCreateKubernetesClusterAsync cannot be called on a nil Organization receiver") } - internalSettings, err := clusterData.toCseClusterSettingsInternal(*org) + cseSubcomponents, err := getCseComponentsVersions(clusterSettings.CseVersion) if err != nil { - return "", fmt.Errorf("error creating the CSE Kubernetes cluster: %s", err) + return "", err } - payload, err := internalSettings.getUnmarshaledRdePayload() + internalSettings, err := clusterSettings.toCseClusterSettingsInternal(*org) if err != nil { - return "", err + return "", fmt.Errorf("error creating the CSE Kubernetes cluster: %s", err) } - cseSubcomponents, err := getCseComponentsVersions(clusterData.CseVersion) + payload, err := internalSettings.getUnmarshaledRdePayload() if err != nil { return "", err } @@ -195,6 +195,7 @@ func (cluster *CseKubernetesCluster) UpdateControlPlane(input CseControlPlaneUpd // cached to avoid querying VCD again multiple times. // If refreshOvas=true, this cache is cleared out and this method will query VCD for every vApp Template again. // Therefore, the refreshOvas flag should be set to true only when VCD has new OVAs that need to be considered or after a cluster upgrade. +// NOTE: Any refresh operation from other methods will cause the cache to be cleared. func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refreshOvas bool) ([]*types.VAppTemplate, error) { if refreshOvas { cluster.supportedUpgrades = make([]*types.VAppTemplate, 0) @@ -218,6 +219,7 @@ func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refreshOvas bool) ([]* if err != nil { continue // This means it's not a TKGm OVA, or it is not supported, so we skip it } + // The OVA can be used if the TKG version is higher than the actual and the Kubernetes version is at most 1 minor higher. if targetVersions.compareTkgVersion(cluster.TkgVersion.String()) == 1 && targetVersions.kubernetesVersionIsOneMinorHigher(cluster.KubernetesVersion.String()) { cluster.supportedUpgrades = append(cluster.supportedUpgrades, vAppTemplate.VAppTemplate) } @@ -273,10 +275,10 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh // automatically by the CSE Server. v411, err := semver.NewVersion("4.1.1") if err != nil { - return fmt.Errorf("can't update the Kubernetes cluster: %s", err) + return fmt.Errorf("can't update the 'Auto Repair on Errors' flag: %s", err) } if cluster.CseVersion.GreaterThanOrEqual(v411) { - return fmt.Errorf("the 'Auto Repair on Errors' feature can't be changed after the cluster is created since CSE 4.1.1") + return fmt.Errorf("the 'Auto Repair on Errors' flag can't be changed after the cluster is created since CSE 4.1.1") } cluster.capvcdType.Spec.VcdKe.AutoRepairOnErrors = *input.AutoRepairOnErrors } @@ -297,7 +299,7 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh return err } - // We do this loop to increase the chances that the Kubernetes cluster is successfully updated, as the Go SDK is + // We do this loop to increase the chances that the Kubernetes cluster is successfully updated, as this method will be // "fighting" with the CSE Server retries := 0 maxRetries := 5 From 905ac8b6e771d8e2f028fe77efcd8305fb1fcfb4 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 16:07:33 +0100 Subject: [PATCH 082/115] Self-review and fix tests Signed-off-by: abarreiro --- govcd/cse/4.1/capiyaml_cluster.tmpl | 2 +- govcd/cse/4.1/capiyaml_mhc.tmpl | 2 +- govcd/cse/4.1/capiyaml_workerpool.tmpl | 2 +- govcd/cse/4.2.0/capiyaml_cluster.tmpl | 2 +- govcd/cse/4.2.0/capiyaml_mhc.tmpl | 2 +- govcd/cse/4.2.0/capiyaml_workerpool.tmpl | 2 +- govcd/cse_internal.go | 76 ++++++++++++------------ govcd/cse_test.go | 19 ++++-- 8 files changed, 57 insertions(+), 50 deletions(-) diff --git a/govcd/cse/4.1/capiyaml_cluster.tmpl b/govcd/cse/4.1/capiyaml_cluster.tmpl index 16a676ae1..92a6bb538 100644 --- a/govcd/cse/4.1/capiyaml_cluster.tmpl +++ b/govcd/cse/4.1/capiyaml_cluster.tmpl @@ -150,4 +150,4 @@ spec: criSocket: /run/containerd/containerd.sock kubeletExtraArgs: eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% - cloud-provider: external \ No newline at end of file + cloud-provider: external diff --git a/govcd/cse/4.1/capiyaml_mhc.tmpl b/govcd/cse/4.1/capiyaml_mhc.tmpl index d31e4c3ec..5d6d912ba 100644 --- a/govcd/cse/4.1/capiyaml_mhc.tmpl +++ b/govcd/cse/4.1/capiyaml_mhc.tmpl @@ -19,4 +19,4 @@ spec: timeout: "{{.NodeUnknownTimeout}}" - type: Ready status: "False" - timeout: "{{.NodeNotReadyTimeout}}" \ No newline at end of file + timeout: "{{.NodeNotReadyTimeout}}" diff --git a/govcd/cse/4.1/capiyaml_workerpool.tmpl b/govcd/cse/4.1/capiyaml_workerpool.tmpl index e2292c7d7..9b7ffbe0c 100644 --- a/govcd/cse/4.1/capiyaml_workerpool.tmpl +++ b/govcd/cse/4.1/capiyaml_workerpool.tmpl @@ -38,4 +38,4 @@ spec: kind: VCDMachineTemplate name: "{{.NodePoolName}}" namespace: "{{.TargetNamespace}}" - version: "{{.KubernetesVersion}}" \ No newline at end of file + version: "{{.KubernetesVersion}}" diff --git a/govcd/cse/4.2.0/capiyaml_cluster.tmpl b/govcd/cse/4.2.0/capiyaml_cluster.tmpl index 16a676ae1..92a6bb538 100644 --- a/govcd/cse/4.2.0/capiyaml_cluster.tmpl +++ b/govcd/cse/4.2.0/capiyaml_cluster.tmpl @@ -150,4 +150,4 @@ spec: criSocket: /run/containerd/containerd.sock kubeletExtraArgs: eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% - cloud-provider: external \ No newline at end of file + cloud-provider: external diff --git a/govcd/cse/4.2.0/capiyaml_mhc.tmpl b/govcd/cse/4.2.0/capiyaml_mhc.tmpl index d31e4c3ec..5d6d912ba 100644 --- a/govcd/cse/4.2.0/capiyaml_mhc.tmpl +++ b/govcd/cse/4.2.0/capiyaml_mhc.tmpl @@ -19,4 +19,4 @@ spec: timeout: "{{.NodeUnknownTimeout}}" - type: Ready status: "False" - timeout: "{{.NodeNotReadyTimeout}}" \ No newline at end of file + timeout: "{{.NodeNotReadyTimeout}}" diff --git a/govcd/cse/4.2.0/capiyaml_workerpool.tmpl b/govcd/cse/4.2.0/capiyaml_workerpool.tmpl index e2292c7d7..9b7ffbe0c 100644 --- a/govcd/cse/4.2.0/capiyaml_workerpool.tmpl +++ b/govcd/cse/4.2.0/capiyaml_workerpool.tmpl @@ -38,4 +38,4 @@ spec: kind: VCDMachineTemplate name: "{{.NodePoolName}}" namespace: "{{.TargetNamespace}}" - version: "{{.KubernetesVersion}}" \ No newline at end of file + version: "{{.KubernetesVersion}}" diff --git a/govcd/cse_internal.go b/govcd/cse_internal.go index dcd080d24..c60ef9a08 100644 --- a/govcd/cse_internal.go +++ b/govcd/cse_internal.go @@ -21,14 +21,14 @@ var cseFiles embed.FS // a CSE Kubernetes cluster, by using the receiver information. This method uses all the Go Templates stored in cseFiles func (clusterSettings *cseClusterSettingsInternal) getUnmarshaledRdePayload() (map[string]interface{}, error) { if clusterSettings == nil { - return nil, fmt.Errorf("the receiver cluster settings is nil") + return nil, fmt.Errorf("the receiver CSE Kubernetes cluster settings object is nil") } capiYaml, err := clusterSettings.generateCapiYamlAsJsonString() if err != nil { return nil, err } - args := map[string]string{ + templateArgs := map[string]string{ "Name": clusterSettings.Name, "Org": clusterSettings.OrganizationName, "VcdUrl": clusterSettings.VcdUrl, @@ -41,27 +41,27 @@ func (clusterSettings *cseClusterSettingsInternal) getUnmarshaledRdePayload() (m } if clusterSettings.DefaultStorageClass.StorageProfileName != "" { - args["DefaultStorageClassStorageProfile"] = clusterSettings.DefaultStorageClass.StorageProfileName - args["DefaultStorageClassName"] = clusterSettings.DefaultStorageClass.Name - args["DefaultStorageClassUseDeleteReclaimPolicy"] = strconv.FormatBool(clusterSettings.DefaultStorageClass.UseDeleteReclaimPolicy) - args["DefaultStorageClassFileSystem"] = clusterSettings.DefaultStorageClass.Filesystem + templateArgs["DefaultStorageClassStorageProfile"] = clusterSettings.DefaultStorageClass.StorageProfileName + templateArgs["DefaultStorageClassName"] = clusterSettings.DefaultStorageClass.Name + templateArgs["DefaultStorageClassUseDeleteReclaimPolicy"] = strconv.FormatBool(clusterSettings.DefaultStorageClass.UseDeleteReclaimPolicy) + templateArgs["DefaultStorageClassFileSystem"] = clusterSettings.DefaultStorageClass.Filesystem } - rdeTmpl, err := getCseTemplate(clusterSettings.CseVersion, "rde") + rdeTemplate, err := getCseTemplate(clusterSettings.CseVersion, "rde") if err != nil { return nil, err } - capvcdEmpty := template.Must(template.New(clusterSettings.Name).Parse(rdeTmpl)) + rdePayload := template.Must(template.New(clusterSettings.Name).Parse(rdeTemplate)) buf := &bytes.Buffer{} - if err := capvcdEmpty.Execute(buf, args); err != nil { - return nil, fmt.Errorf("could not render the Go template with the CAPVCD JSON: %s", err) + if err := rdePayload.Execute(buf, templateArgs); err != nil { + return nil, fmt.Errorf("could not render the Go template with the RDE JSON: %s", err) } var result interface{} err = json.Unmarshal(buf.Bytes(), &result) if err != nil { - return nil, fmt.Errorf("could not generate a correct CAPVCD JSON: %s", err) + return nil, fmt.Errorf("could not generate a correct RDE payload: %s", err) } return result.(map[string]interface{}), nil @@ -74,14 +74,14 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( return "", fmt.Errorf("the receiver cluster settings is nil") } - clusterTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_cluster") + capiYamlTemplate, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_cluster") if err != nil { return "", err } // This YAML snippet contains special strings, such as "%,", that render wrong using the Go template engine - sanitizedTemplate := strings.NewReplacer("%", "%%").Replace(clusterTmpl) - capiYamlEmpty := template.Must(template.New(clusterSettings.Name + "-cluster").Parse(sanitizedTemplate)) + sanitizedCapiYamlTemplate := strings.NewReplacer("%", "%%").Replace(capiYamlTemplate) + capiYaml := template.Must(template.New(clusterSettings.Name + "-cluster").Parse(sanitizedCapiYamlTemplate)) nodePoolYaml, err := clusterSettings.generateWorkerPoolsYaml() if err != nil { @@ -93,7 +93,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( return "", err } - args := map[string]string{ + templateArgs := map[string]string{ "ClusterName": clusterSettings.Name, "TargetNamespace": clusterSettings.Name + "-ns", "TkrVersion": clusterSettings.TkgVersionBundle.TkrVersion, @@ -123,7 +123,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( } buf := &bytes.Buffer{} - if err := capiYamlEmpty.Execute(buf, args); err != nil { + if err := capiYaml.Execute(buf, templateArgs); err != nil { return "", fmt.Errorf("could not generate a correct CAPI YAML: %s", err) } @@ -140,7 +140,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( enc.SetEscapeHTML(false) err = enc.Encode(prettyYaml) if err != nil { - return "", fmt.Errorf("could not encode the CAPI YAML into JSON: %s", err) + return "", fmt.Errorf("could not encode the CAPI YAML into a JSON string: %s", err) } // Removes trailing quotes from the final JSON string @@ -151,46 +151,46 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( // the standard YAML separator (---), but does not add one at the end. func (clusterSettings *cseClusterSettingsInternal) generateWorkerPoolsYaml() (string, error) { if clusterSettings == nil { - return "", fmt.Errorf("the receiver cluster settings is nil") + return "", fmt.Errorf("the receiver CSE Kubernetes cluster settings object is nil") } - workerPoolTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_workerpool") + workerPoolsTemplate, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_workerpool") if err != nil { return "", err } - nodePoolEmptyTmpl := template.Must(template.New(clusterSettings.Name + "-worker-pool").Parse(workerPoolTmpl)) + workerPools := template.Must(template.New(clusterSettings.Name + "-worker-pool").Parse(workerPoolsTemplate)) resultYaml := "" buf := &bytes.Buffer{} // We can have many Worker Pools, we build a YAML object for each one of them. - for i, workerPool := range clusterSettings.WorkerPools { + for i, wp := range clusterSettings.WorkerPools { // Check the correctness of the Compute Policies in the node pool block - if workerPool.PlacementPolicyName != "" && workerPool.VGpuPolicyName != "" { - return "", fmt.Errorf("the worker pool '%s' should have either a Placement Policy or a vGPU Policy, not both", workerPool.Name) + if wp.PlacementPolicyName != "" && wp.VGpuPolicyName != "" { + return "", fmt.Errorf("the Worker Pool '%s' should have either a Placement Policy or a vGPU Policy, not both", wp.Name) } - placementPolicy := workerPool.PlacementPolicyName - if workerPool.VGpuPolicyName != "" { + placementPolicy := wp.PlacementPolicyName + if wp.VGpuPolicyName != "" { // For convenience, we just use one of the variables as both cannot be set at same time - placementPolicy = workerPool.VGpuPolicyName + placementPolicy = wp.VGpuPolicyName } - if err := nodePoolEmptyTmpl.Execute(buf, map[string]string{ + if err := workerPools.Execute(buf, map[string]string{ "ClusterName": clusterSettings.Name, - "NodePoolName": workerPool.Name, + "NodePoolName": wp.Name, "TargetNamespace": clusterSettings.Name + "-ns", "Catalog": clusterSettings.CatalogName, "VAppTemplate": clusterSettings.KubernetesTemplateOvaName, - "NodePoolSizingPolicy": workerPool.SizingPolicyName, + "NodePoolSizingPolicy": wp.SizingPolicyName, "NodePoolPlacementPolicy": placementPolicy, // Can be either Placement or vGPU policy - "NodePoolStorageProfile": workerPool.StorageProfileName, - "NodePoolDiskSize": fmt.Sprintf("%dGi", workerPool.DiskSizeGi), - "NodePoolEnableGpu": strconv.FormatBool(workerPool.VGpuPolicyName != ""), - "NodePoolMachineCount": strconv.Itoa(workerPool.MachineCount), + "NodePoolStorageProfile": wp.StorageProfileName, + "NodePoolDiskSize": fmt.Sprintf("%dGi", wp.DiskSizeGi), + "NodePoolEnableGpu": strconv.FormatBool(wp.VGpuPolicyName != ""), + "NodePoolMachineCount": strconv.Itoa(wp.MachineCount), "KubernetesVersion": clusterSettings.TkgVersionBundle.KubernetesVersion, }); err != nil { - return "", fmt.Errorf("could not generate a correct Node Pool YAML: %s", err) + return "", fmt.Errorf("could not generate a correct Worker Pool '%s' YAML block: %s", wp.Name, err) } resultYaml += fmt.Sprintf("%s\n", buf.String()) if i < len(clusterSettings.WorkerPools)-1 { @@ -205,7 +205,7 @@ func (clusterSettings *cseClusterSettingsInternal) generateWorkerPoolsYaml() (st // The generated YAML does not contain a separator (---) at the end. func (clusterSettings *cseClusterSettingsInternal) generateMachineHealthCheckYaml() (string, error) { if clusterSettings == nil { - return "", fmt.Errorf("the receiver cluster settings is nil") + return "", fmt.Errorf("the receiver CSE Kubernetes cluster settings object is nil") } if clusterSettings.VcdKeConfig.NodeStartupTimeout == "" && @@ -215,15 +215,15 @@ func (clusterSettings *cseClusterSettingsInternal) generateMachineHealthCheckYam return "", nil } - mhcTmpl, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_mhc") + mhcTemplate, err := getCseTemplate(clusterSettings.CseVersion, "capiyaml_mhc") if err != nil { return "", err } - mhcEmptyTmpl := template.Must(template.New(clusterSettings.Name + "-mhc").Parse(mhcTmpl)) + machineHealthCheck := template.Must(template.New(clusterSettings.Name + "-mhc").Parse(mhcTemplate)) buf := &bytes.Buffer{} - if err := mhcEmptyTmpl.Execute(buf, map[string]string{ + if err := machineHealthCheck.Execute(buf, map[string]string{ "ClusterName": clusterSettings.Name, "TargetNamespace": clusterSettings.Name + "-ns", // With the 'percentage' suffix diff --git a/govcd/cse_test.go b/govcd/cse_test.go index e9c7fef56..b4a5d4b17 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -114,6 +114,16 @@ func (vcd *TestVCD) Test_Cse(check *C) { AutoRepairOnErrors: true, } cluster, err := org.CseCreateKubernetesCluster(clusterSettings, 0) + + // We assure that the cluster gets always deleted, even if the creation failed. + // Deletion process only needs the cluster ID + defer func() { + check.Assert(cluster, NotNil) + check.Assert(cluster.ID, Not(Equals), "") + err = cluster.Delete(0) + check.Assert(err, IsNil) + }() + check.Assert(err, IsNil) assertCseClusterCreation(check, cluster, clusterSettings, tkgBundle) @@ -169,10 +179,10 @@ func (vcd *TestVCD) Test_Cse(check *C) { } check.Assert(foundWorkerPool, Equals, true) - // Update control plane from 1 node to 2 - err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 2}, true) + // Update control plane from 1 node to 3 (needs to be an odd number) + err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 3}, true) check.Assert(err, IsNil) - check.Assert(cluster.ControlPlane.MachineCount, Equals, 2) + check.Assert(cluster.ControlPlane.MachineCount, Equals, 3) // Turn off the node health check err = cluster.SetNodeHealthCheck(false, true) @@ -222,9 +232,6 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(cluster.ControlPlane.MachineCount, Equals, 1) check.Assert(cluster.WorkerPools[0].MachineCount, Equals, 1) check.Assert(cluster.WorkerPools[1].MachineCount, Equals, 0) - - err = cluster.Delete(0) - check.Assert(err, IsNil) } func assertCseClusterCreation(check *C, createdCluster *CseKubernetesCluster, settings CseClusterSettings, expectedKubernetesData tkgVersionBundle) { From 941bef102814a73084a4f4a343e79951b09f4bc6 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 16:27:29 +0100 Subject: [PATCH 083/115] Self-review and fix tests Signed-off-by: abarreiro --- govcd/defined_entity.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/govcd/defined_entity.go b/govcd/defined_entity.go index f1797e24c..0ed0c0425 100644 --- a/govcd/defined_entity.go +++ b/govcd/defined_entity.go @@ -429,8 +429,8 @@ func createRde(client *Client, entity types.DefinedEntity, tenantContext *Tenant return &task, nil } -// getRdeFromTask polls VCD for a given amount of tries, to search for the RDE in the given Task that should have -// been created as result of creating that RDE. +// getRdeFromTask gets the Runtime Defined Entity from a given Task. This method is useful after RDE creation, as +// the API just returns a Task with the RDE details inside. func getRdeFromTask(client *Client, task *Task) (*DefinedEntity, error) { if task.Task == nil { return nil, fmt.Errorf("could not retrieve the RDE from task, as it is nil") From fc3ce743d46754f604f1b035cbad32d0839bbb8c Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 16:51:33 +0100 Subject: [PATCH 084/115] Self-review and fix tests Signed-off-by: abarreiro --- govcd/defined_entity.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/govcd/defined_entity.go b/govcd/defined_entity.go index 0ed0c0425..1539d3413 100644 --- a/govcd/defined_entity.go +++ b/govcd/defined_entity.go @@ -422,7 +422,7 @@ func createRde(client *Client, entity types.DefinedEntity, tenantContext *Tenant return nil, err } - task, err := client.OpenApiPutItemAsync(apiVersion, urlRef, nil, entity, getTenantContextHeader(tenantContext)) + task, err := client.OpenApiPostItemAsyncWithHeaders(apiVersion, urlRef, nil, entity, getTenantContextHeader(tenantContext)) if err != nil { return nil, err } From 62291e96db2dd7e0f49de010693b204737692ab5 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 17:07:48 +0100 Subject: [PATCH 085/115] Fix RDE get Signed-off-by: abarreiro --- govcd/defined_entity.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/govcd/defined_entity.go b/govcd/defined_entity.go index 1539d3413..2c8d74fa0 100644 --- a/govcd/defined_entity.go +++ b/govcd/defined_entity.go @@ -426,6 +426,11 @@ func createRde(client *Client, entity types.DefinedEntity, tenantContext *Tenant if err != nil { return nil, err } + // The refresh is needed as the task only has the HREF at the moment + err = task.Refresh() + if err != nil { + return nil, err + } return &task, nil } From 8887f6eb3fe32f520ecebb5397d49b44f6f97c5c Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 22 Feb 2024 20:17:24 +0100 Subject: [PATCH 086/115] Fix test Signed-off-by: abarreiro --- govcd/cse_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index b4a5d4b17..bea1b68ec 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -230,8 +230,13 @@ func (vcd *TestVCD) Test_Cse(check *C) { }, true) check.Assert(err, IsNil) check.Assert(cluster.ControlPlane.MachineCount, Equals, 1) - check.Assert(cluster.WorkerPools[0].MachineCount, Equals, 1) - check.Assert(cluster.WorkerPools[1].MachineCount, Equals, 0) + for _, pool := range cluster.WorkerPools { + if pool.Name == "new-pool" { + check.Assert(pool.MachineCount, Equals, 0) + } else { + check.Assert(pool.MachineCount, Equals, 1) + } + } } func assertCseClusterCreation(check *C, createdCluster *CseKubernetesCluster, settings CseClusterSettings, expectedKubernetesData tkgVersionBundle) { From cba5468cd412000a5e95b5179bcdd6193c0c9e9f Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 10:23:24 +0100 Subject: [PATCH 087/115] Fix unit Signed-off-by: abarreiro --- govcd/cse_internal_unit_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/govcd/cse_internal_unit_test.go b/govcd/cse_internal_unit_test.go index 0e0e46225..a49be83f7 100644 --- a/govcd/cse_internal_unit_test.go +++ b/govcd/cse_internal_unit_test.go @@ -209,7 +209,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) }, }, }, - wantErr: "the worker pool 'node-pool-1' should have either a Placement Policy or a vGPU Policy, not both", + wantErr: "the Worker Pool 'node-pool-1' should have either a Placement Policy or a vGPU Policy, not both", }, } for _, tt := range tests { From e13fdaf7f7c246368022b859a396534cae6e3467 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 10:44:48 +0100 Subject: [PATCH 088/115] Fix unit Signed-off-by: abarreiro --- govcd/cse_yaml_unit_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 24b321966..a6c2b7143 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -208,7 +208,7 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { } // We call the function to update the old pools with the new ones - newReplicas := 66 + newReplicas := 67 newControlPlane := CseControlPlaneUpdateInput{ MachineCount: newReplicas, } From bd9c75e64f68940ab375a26cd374d3092f4691e7 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 10:46:40 +0100 Subject: [PATCH 089/115] Fix unit Signed-off-by: abarreiro --- govcd/cse_yaml_unit_test.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index a6c2b7143..1793e8811 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -111,8 +111,7 @@ func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { if err != nil { t.Fatalf("could not unmarshal CAPI YAML test file: %s", err) } - // We explore the YAML documents to get the OVA template name that will be updated - // with the new one. + // We explore the YAML documents to get the current Worker pool oldNodePools := map[string]CseWorkerPoolUpdateInput{} for _, document := range yamlDocs { if document["kind"] != "MachineDeployment" { @@ -191,8 +190,7 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { if err != nil { t.Fatalf("could not unmarshal CAPI YAML test file: %s", err) } - // We explore the YAML documents to get the OVA template name that will be updated - // with the new one. + // We explore the YAML documents to get the current Control plane oldControlPlane := CseControlPlaneUpdateInput{} for _, document := range yamlDocs { if document["kind"] != "KubeadmControlPlane" { @@ -207,7 +205,7 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { t.Fatalf("didn't get any valid Control Plane") } - // We call the function to update the old pools with the new ones + // We call the function to update the control plane newReplicas := 67 newControlPlane := CseControlPlaneUpdateInput{ MachineCount: newReplicas, @@ -217,7 +215,7 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { t.Fatalf("%s", err) } - // The worker pools should have now the new details updated + // The control plane should have now the new details updated for _, document := range yamlDocs { if document["kind"] != "KubeadmControlPlane" { continue @@ -238,6 +236,15 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { if err == nil { t.Fatal("Expected an error, but got none") } + + newReplicas = 2 // Should be odd, hence fails + newControlPlane = CseControlPlaneUpdateInput{ + MachineCount: newReplicas, + } + err = cseUpdateControlPlaneInYaml(yamlDocs, newControlPlane) + if err == nil { + t.Fatal("Expected an error, but got none") + } } // Test_cseUpdateNodeHealthCheckInYaml tests the update process of the Machine Health Check capabilities in a CAPI YAML. From c8ad117490c25693ffc2b81fc717402238eab097 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 10:54:23 +0100 Subject: [PATCH 090/115] self-review Signed-off-by: abarreiro --- govcd/cse_internal_unit_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/govcd/cse_internal_unit_test.go b/govcd/cse_internal_unit_test.go index a49be83f7..5b1638008 100644 --- a/govcd/cse_internal_unit_test.go +++ b/govcd/cse_internal_unit_test.go @@ -12,7 +12,7 @@ import ( // Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { - v41, err := semver.NewVersion("4.2.0") + cseVersion, err := semver.NewVersion("4.2.0") if err != nil { t.Fatalf("%s", err) } @@ -34,7 +34,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { name: "correct YAML without optionals", input: cseClusterSettingsInternal{ - CseVersion: *v41, + CseVersion: *cseVersion, Name: "test1", OrganizationName: "tenant_org", VdcName: "tenant_vdc", @@ -83,7 +83,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { name: "correct YAML without MachineHealthCheck", input: cseClusterSettingsInternal{ - CseVersion: *v41, + CseVersion: *cseVersion, Name: "test1", OrganizationName: "tenant_org", VdcName: "tenant_vdc", @@ -135,7 +135,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { name: "correct YAML with every possible option", input: cseClusterSettingsInternal{ - CseVersion: *v41, + CseVersion: *cseVersion, Name: "test1", OrganizationName: "tenant_org", VdcName: "tenant_vdc", @@ -196,7 +196,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { name: "wrong YAML with both Placement and vGPU policies in a Worker Pool", input: cseClusterSettingsInternal{ - CseVersion: *v41, + CseVersion: *cseVersion, WorkerPools: []cseWorkerPoolSettingsInternal{ { Name: "node-pool-1", From b23f13c7dd229892458a509e10324178d031c21e Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 11:05:57 +0100 Subject: [PATCH 091/115] self-review Signed-off-by: abarreiro --- govcd/cse.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index c390e0afa..9b9cdd6b8 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -165,7 +165,8 @@ func (cluster *CseKubernetesCluster) GetKubeconfig(refresh bool) (string, error) return result.Capvcd.Status.Capvcd.Private.KubeConfig, nil } -// UpdateWorkerPools executes an update on the receiver cluster to change the existing worker pools. +// UpdateWorkerPools executes an update on the receiver cluster to change the existing Worker Pools. +// The input is a map where the key is the Worker pool unique name, and the value is the update payload for that Worker Pool. // If refresh=true, it retrieves the latest state of the cluster from VCD before updating. // WARNING: At least one worker pool must have one or more nodes running, otherwise the cluster will be left in an unusable state. func (cluster *CseKubernetesCluster) UpdateWorkerPools(input map[string]CseWorkerPoolUpdateInput, refresh bool) error { @@ -174,7 +175,7 @@ func (cluster *CseKubernetesCluster) UpdateWorkerPools(input map[string]CseWorke }, refresh) } -// AddWorkerPools executes an update on the receiver cluster to add new worker pools. +// AddWorkerPools executes an update on the receiver cluster to add new Worker Pools. // If refresh=true, it retrieves the latest state of the cluster from VCD before updating. func (cluster *CseKubernetesCluster) AddWorkerPools(input []CseWorkerPoolSettings, refresh bool) error { return cluster.Update(CseClusterUpdateInput{ From a28f014bb566e1f9a29d3ad4e7f2e78b323a5bbf Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 11:25:06 +0100 Subject: [PATCH 092/115] self-review Signed-off-by: abarreiro --- govcd/cse_util.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 7a1093573..53264f21b 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -796,6 +796,12 @@ func (tkgVersions tkgVersionBundle) compareTkgVersion(tkgVersion string) int { // kubernetesVersionIsOneMinorHigher returns true only if the receiver Kubernetes version is exactly one minor version higher // than the given input version, being the minor digit the 'Y' in 'X.Y.Z'. +// Examples: +// * "1.19.2".kubernetesVersionIsOneMinorHigher("1.18.7") = true +// * "1.19.10".kubernetesVersionIsOneMinorHigher("1.18.0") = true +// * "1.20.2".kubernetesVersionIsOneMinorHigher("1.18.7") = false +// * "1.21.2".kubernetesVersionIsOneMinorHigher("1.18.7") = false +// * "1.18.0".kubernetesVersionIsOneMinorHigher("1.18.7") = false func (tkgVersions tkgVersionBundle) kubernetesVersionIsOneMinorHigher(kubernetesVersion string) bool { receiverVersion, err := semver.NewVersion(tkgVersions.KubernetesVersion) if err != nil { From 6ff4f3c3e657b294489b1b6ea0a7d328590b175c Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 11:25:35 +0100 Subject: [PATCH 093/115] self-review Signed-off-by: abarreiro --- govcd/cse_util.go | 1 + 1 file changed, 1 insertion(+) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 53264f21b..de8a38bdb 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -796,6 +796,7 @@ func (tkgVersions tkgVersionBundle) compareTkgVersion(tkgVersion string) int { // kubernetesVersionIsOneMinorHigher returns true only if the receiver Kubernetes version is exactly one minor version higher // than the given input version, being the minor digit the 'Y' in 'X.Y.Z'. +// Any malformed version returns false. // Examples: // * "1.19.2".kubernetesVersionIsOneMinorHigher("1.18.7") = true // * "1.19.10".kubernetesVersionIsOneMinorHigher("1.18.0") = true From ebc292ee960d11c5bacf57dc19ddb6525b314a9a Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 11:35:47 +0100 Subject: [PATCH 094/115] Add more unit tests Signed-off-by: abarreiro --- govcd/cse_util_unit_test.go | 106 ++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index 1d9139eea..d201fd61b 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -115,6 +115,112 @@ func Test_getTkgVersionBundleFromVAppTemplate(t *testing.T) { } } +func Test_tkgVersionBundle_compareTkgVersion(t *testing.T) { + tests := []struct { + name string + receiverTkgVersion string + comparedTkgVersion string + want int + }{ + { + name: "same TKG version", + receiverTkgVersion: "v1.4.3", + comparedTkgVersion: "v1.4.3", + want: 0, + }, + { + name: "receiver TKG version is higher", + receiverTkgVersion: "v1.4.4", + comparedTkgVersion: "v1.4.3", + want: 1, + }, + { + name: "receiver TKG version is lower", + receiverTkgVersion: "v1.4.2", + comparedTkgVersion: "v1.4.3", + want: -1, + }, + { + name: "receiver TKG version is wrong", + receiverTkgVersion: "foo", + comparedTkgVersion: "v1.4.3", + want: -2, + }, + { + name: "compared TKG version is wrong", + receiverTkgVersion: "v1.4.3", + comparedTkgVersion: "foo", + want: -2, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tkgVersions := tkgVersionBundle{ + TkgVersion: tt.receiverTkgVersion, + } + if got := tkgVersions.compareTkgVersion(tt.comparedTkgVersion); got != tt.want { + t.Errorf("kubernetesVersionIsOneMinorHigher() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_tkgVersionBundle_kubernetesVersionIsOneMinorHigher(t *testing.T) { + tests := []struct { + name string + receiverKubernetesVersion string + comparedKubernetesVersion string + want bool + }{ + { + name: "same Kubernetes versions", + receiverKubernetesVersion: "1.20.2+vmware.1", + comparedKubernetesVersion: "1.20.2+vmware.1", + want: false, + }, + { + name: "one Kubernetes minor higher", + receiverKubernetesVersion: "1.21.9+vmware.1", + comparedKubernetesVersion: "1.20.2+vmware.1", + want: true, + }, + { + name: "one Kubernetes minor lower", + receiverKubernetesVersion: "1.19.9+vmware.1", + comparedKubernetesVersion: "1.20.2+vmware.1", + want: false, + }, + { + name: "several Kubernetes minors higher", + receiverKubernetesVersion: "1.22.9+vmware.1", + comparedKubernetesVersion: "1.20.2+vmware.1", + want: false, + }, + { + name: "wrong receiver Kubernetes version", + receiverKubernetesVersion: "foo", + comparedKubernetesVersion: "1.20.2+vmware.1", + want: false, + }, + { + name: "wrong compared Kubernetes version", + receiverKubernetesVersion: "1.20.2+vmware.1", + comparedKubernetesVersion: "foo", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tkgVersions := tkgVersionBundle{ + KubernetesVersion: tt.receiverKubernetesVersion, + } + if got := tkgVersions.kubernetesVersionIsOneMinorHigher(tt.comparedKubernetesVersion); got != tt.want { + t.Errorf("kubernetesVersionIsOneMinorHigher() = %v, want %v", got, tt.want) + } + }) + } +} + func Test_getCseTemplate(t *testing.T) { v40, err := semver.NewVersion("4.0") if err != nil { From eb239a3b2bbca412d6aeecf5aac7ba5fc236a326 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 16:17:44 +0100 Subject: [PATCH 095/115] Remove minutes from timeouts Signed-off-by: abarreiro --- govcd/cse.go | 14 +++++++------- govcd/cse_util.go | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 9b9cdd6b8..b3534aa8a 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -19,13 +19,13 @@ import ( // // If the cluster is created correctly, returns all the available data in CseKubernetesCluster or an error if some of the fields // of the created cluster cannot be calculated or retrieved. -func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterSettings, timeoutMinutes time.Duration) (*CseKubernetesCluster, error) { +func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterSettings, timeout time.Duration) (*CseKubernetesCluster, error) { clusterId, err := org.CseCreateKubernetesClusterAsync(clusterData) if err != nil { return nil, err } - err = waitUntilClusterIsProvisioned(org.client, clusterId, timeoutMinutes) + err = waitUntilClusterIsProvisioned(org.client, clusterId, timeout) if err != nil { return &CseKubernetesCluster{ID: clusterId}, err } @@ -334,14 +334,14 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh return cluster.Refresh() } -// Delete deletes a CSE Kubernetes cluster, waiting the specified amount of minutes. If the timeout is reached, this method +// Delete deletes a CSE Kubernetes cluster, waiting the specified amount of time. If the timeout is reached, this method // returns an error, even if the cluster is already marked for deletion. -func (cluster *CseKubernetesCluster) Delete(timeoutMinutes time.Duration) error { +func (cluster *CseKubernetesCluster) Delete(timeout time.Duration) error { var elapsed time.Duration start := time.Now() markForDelete := false forceDelete := false - for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies timeoutMinutes=0, we wait forever + for elapsed <= timeout || timeout == 0 { // If the user specifies timeout=0, we wait forever rde, err := getRdeById(cluster.client, cluster.ID) if err != nil { if ContainsNotFound(err) { @@ -373,7 +373,7 @@ func (cluster *CseKubernetesCluster) Delete(timeoutMinutes time.Duration) error // We give a hint to the user about the deletion process result if markForDelete && forceDelete { - return fmt.Errorf("timeout of %v minutes reached, the cluster was successfully marked for deletion but was not removed in time", timeoutMinutes) + return fmt.Errorf("timeout of %s reached, the cluster was successfully marked for deletion but was not removed in time", timeout) } - return fmt.Errorf("timeout of %v minutes reached, the cluster was not marked for deletion, please try again", timeoutMinutes) + return fmt.Errorf("timeout of %s reached, the cluster was not marked for deletion, please try again", timeout) } diff --git a/govcd/cse_util.go b/govcd/cse_util.go index de8a38bdb..b85e59a32 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -461,18 +461,18 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu return result, nil } -// waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if timeoutMinutes = 0) +// waitUntilClusterIsProvisioned waits for the Kubernetes cluster to be in "provisioned" state, either indefinitely (if timeout = 0) // or until the timeout is reached. // If one of the states of the cluster at a given point is "error", this function also checks whether the cluster has the "AutoRepairOnErrors" flag enabled, // so it keeps waiting if it's true. // If timeout is reached before the cluster is in "provisioned" state, it returns an error. -func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinutes time.Duration) error { +func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeout time.Duration) error { var elapsed time.Duration sleepTime := 10 start := time.Now() capvcd := &types.Capvcd{} - for elapsed <= timeoutMinutes*time.Minute || timeoutMinutes == 0 { // If the user specifies timeoutMinutes=0, we wait forever + for elapsed <= timeout || timeout == 0 { // If the user specifies timeout=0, we wait forever rde, err := getRdeById(client, clusterId) if err != nil { return err @@ -507,7 +507,7 @@ func waitUntilClusterIsProvisioned(client *Client, clusterId string, timeoutMinu elapsed = time.Since(start) time.Sleep(time.Duration(sleepTime) * time.Second) } - return fmt.Errorf("timeout of %d minutes reached, latest cluster state obtained was '%s'", timeoutMinutes, capvcd.Status.VcdKe.State) + return fmt.Errorf("timeout of %s reached, latest cluster state obtained was '%s'", timeout, capvcd.Status.VcdKe.State) } // validate validates the receiver CseClusterSettings. Returns an error if any of the fields is empty or wrong. From 5202f9d1805d89ff6964309270590c8bcb5fdf3a Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 16:21:14 +0100 Subject: [PATCH 096/115] Fix changelog Signed-off-by: abarreiro --- .changes/v2.23.0/645-features.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.changes/v2.23.0/645-features.md b/.changes/v2.23.0/645-features.md index 3b4537c2b..38ed2dc6a 100644 --- a/.changes/v2.23.0/645-features.md +++ b/.changes/v2.23.0/645-features.md @@ -1,19 +1,19 @@ -* Added a new type `CseKubernetesCluster` to manage Container Service Extension Kubernetes clusters [GH-645] -* Added new methods `Org.CseCreateKubernetesCluster` and `Org.CseCreateKubernetesClusterAsync` to create Kubernetes clusters +* Added the type `CseKubernetesCluster` to manage Container Service Extension Kubernetes clusters [GH-645] +* Added methods `Org.CseCreateKubernetesCluster` and `Org.CseCreateKubernetesClusterAsync` to create Kubernetes clusters in a VCD appliance with Container Service Extension installed [GH-645] -* Added new methods `VCDClient.CseGetKubernetesClusterById` and `Org.CseGetKubernetesClustersByName` to retrieve a +* Added methods `VCDClient.CseGetKubernetesClusterById` and `Org.CseGetKubernetesClustersByName` to retrieve a Container Service Extension Kubernetes cluster [GH-645] -* Added a new method `CseKubernetesCluster.GetKubeconfig` to retrieve the *kubeconfig* of a provisioned Container Service +* Added the method `CseKubernetesCluster.GetKubeconfig` to retrieve the *kubeconfig* of a provisioned Container Service Extension Kubernetes cluster [GH-645] -* Added a new method `CseKubernetesCluster.Refresh` to refresh the information and properties of an existing Container +* Added the method `CseKubernetesCluster.Refresh` to refresh the information and properties of an existing Container Service Extension Kubernetes cluster [GH-645] -* Added new methods to update a Container Service Extension Kubernetes cluster: `CseKubernetesCluster.UpdateWorkerPools`, +* Added methods to update a Container Service Extension Kubernetes cluster: `CseKubernetesCluster.UpdateWorkerPools`, `CseKubernetesCluster.AddWorkerPools`, `CseKubernetesCluster.UpdateControlPlane`, `CseKubernetesCluster.UpgradeCluster`, `CseKubernetesCluster.SetNodeHealthCheck` and `CseKubernetesCluster.SetAutoRepairOnErrors` [GH-645] -* Added a new method `CseKubernetesCluster.GetSupportedUpgrades` to retrieve all the valid TKGm OVAs that a given Container +* Added the method `CseKubernetesCluster.GetSupportedUpgrades` to retrieve all the valid TKGm OVAs that a given Container Service Extension Kubernetes cluster can use to be upgraded [GH-645] -* Added a new method `CseKubernetesCluster.Delete` to delete a cluster [GH-645] -* Added new types `CseClusterSettings`, `CseControlPlaneSettings`, `CseWorkerPoolSettings` and `CseDefaultStorageClassSettings` +* Added the method `CseKubernetesCluster.Delete` to delete a cluster [GH-645] +* Added types `CseClusterSettings`, `CseControlPlaneSettings`, `CseWorkerPoolSettings` and `CseDefaultStorageClassSettings` to configure the Container Service Extension Kubernetes clusters creation process [GH-645] -* Added new types `CseClusterUpdateInput`, `CseControlPlaneUpdateInput` and `CseWorkerPoolUpdateInput` to configure the +* Added types `CseClusterUpdateInput`, `CseControlPlaneUpdateInput` and `CseWorkerPoolUpdateInput` to configure the Container Service Extension Kubernetes clusters update process [GH-645] From 036fad24456579d3da2e49612b45ba04acb55f7a Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 26 Feb 2024 16:28:47 +0100 Subject: [PATCH 097/115] Remove built-in YAML library Signed-off-by: abarreiro --- go.mod | 6 +----- go.sum | 4 ++-- govcd/api_vcd_test.go | 2 +- samples/discover/discover.go | 3 +-- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 522439b1b..9ae8e3a92 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/kr/pretty v0.2.1 github.com/peterhellberg/link v1.1.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 - gopkg.in/yaml.v2 v2.2.2 sigs.k8s.io/yaml v1.4.0 ) @@ -20,7 +19,4 @@ require ( golang.org/x/text v0.14.0 ) -replace ( - gopkg.in/check.v1 => github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c - gopkg.in/yaml.v2 => github.com/go-yaml/yaml/v2 v2.2.2 -) +replace gopkg.in/check.v1 => github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c diff --git a/go.sum b/go.sum index 27859793b..8b4be2b5e 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,6 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c h1:3LdnoQiW6yLkxRIwSU3pbYp3zqW1daDgoOcOD09OzJs= github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -github.com/go-yaml/yaml/v2 v2.2.2 h1:uw2m9KuKRscWGAkuyoBGQcZSdibhmuXKSJ3+9Tj3zXc= -github.com/go-yaml/yaml/v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= @@ -26,5 +24,7 @@ golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRj golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/govcd/api_vcd_test.go b/govcd/api_vcd_test.go index 16bc66855..771fe0e2f 100644 --- a/govcd/api_vcd_test.go +++ b/govcd/api_vcd_test.go @@ -17,13 +17,13 @@ import ( "path/filepath" "regexp" "runtime" + "sigs.k8s.io/yaml" "strings" "sync" "testing" "time" . "gopkg.in/check.v1" - "gopkg.in/yaml.v2" "github.com/vmware/go-vcloud-director/v2/types/v56" "github.com/vmware/go-vcloud-director/v2/util" diff --git a/samples/discover/discover.go b/samples/discover/discover.go index 50dfac68f..815bcba8d 100644 --- a/samples/discover/discover.go +++ b/samples/discover/discover.go @@ -46,8 +46,7 @@ import ( "net/url" "os" "path/filepath" - - "gopkg.in/yaml.v2" + "sigs.k8s.io/yaml" "github.com/vmware/go-vcloud-director/v2/govcd" ) From bdcafd6fbed2a09481206d27b1c72e610cbd6dff Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 27 Feb 2024 17:10:00 +0100 Subject: [PATCH 098/115] Add unit tests for getCseComponentsVersions Signed-off-by: abarreiro --- govcd/cse_util.go | 17 ++++++---- govcd/cse_util_unit_test.go | 66 +++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index b85e59a32..371687956 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -16,21 +16,26 @@ import ( ) // getCseComponentsVersions gets the versions of the subcomponents that are part of Container Service Extension. +// NOTE: This function should be updated on every CSE release to update the supported versions. func getCseComponentsVersions(cseVersion semver.Version) (*cseComponentsVersions, error) { - v41, _ := semver.NewVersion("4.1.0") - v42, _ := semver.NewVersion("4.2.0") v43, _ := semver.NewVersion("4.3.0") + v42, _ := semver.NewVersion("4.2.0") + v41, _ := semver.NewVersion("4.1.0") - if cseVersion.GreaterThanOrEqual(v41) && cseVersion.LessThan(v42) { + if cseVersion.GreaterThanOrEqual(v43) { + return nil, fmt.Errorf("the Container Service Extension version '%s' is not supported", cseVersion.String()) + } + if cseVersion.GreaterThanOrEqual(v42) { return &cseComponentsVersions{ VcdKeConfigRdeTypeVersion: "1.1.0", - CapvcdRdeTypeVersion: "1.2.0", + CapvcdRdeTypeVersion: "1.3.0", CseInterfaceVersion: "1.0.0", }, nil - } else if cseVersion.GreaterThanOrEqual(v42) && cseVersion.LessThan(v43) { + } + if cseVersion.GreaterThanOrEqual(v41) { return &cseComponentsVersions{ VcdKeConfigRdeTypeVersion: "1.1.0", - CapvcdRdeTypeVersion: "1.3.0", + CapvcdRdeTypeVersion: "1.2.0", CseInterfaceVersion: "1.0.0", }, nil } diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index d201fd61b..3699a7fbc 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -13,6 +13,72 @@ import ( "testing" ) +func Test_getCseComponentsVersions(t *testing.T) { + tests := []struct { + name string + cseVersion string + want *cseComponentsVersions + wantErr bool + }{ + { + name: "CSE 4.0 is not supported", + cseVersion: "4.0", + wantErr: true, + }, + { + name: "CSE 4.1 is supported", + cseVersion: "4.1", + want: &cseComponentsVersions{ + VcdKeConfigRdeTypeVersion: "1.1.0", + CapvcdRdeTypeVersion: "1.2.0", + CseInterfaceVersion: "1.0.0", + }, + wantErr: false, + }, + { + name: "CSE 4.1.1 is supported", + cseVersion: "4.1.1", + want: &cseComponentsVersions{ + VcdKeConfigRdeTypeVersion: "1.1.0", + CapvcdRdeTypeVersion: "1.2.0", + CseInterfaceVersion: "1.0.0", + }, + wantErr: false, + }, + { + name: "CSE 4.2 is supported", + cseVersion: "4.2", + want: &cseComponentsVersions{ + VcdKeConfigRdeTypeVersion: "1.1.0", + CapvcdRdeTypeVersion: "1.3.0", + CseInterfaceVersion: "1.0.0", + }, + wantErr: false, + }, + { + name: "CSE 4.3 is not supported", + cseVersion: "4.3", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + version, err := semver.NewVersion(tt.cseVersion) + if err != nil { + t.Fatalf("could not parse test version: %s", err) + } + got, err := getCseComponentsVersions(*version) + if (err != nil) != tt.wantErr { + t.Errorf("getCseComponentsVersions() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getCseComponentsVersions() got = %v, want %v", got, tt.want) + } + }) + } +} + // Test_getTkgVersionBundleFromVAppTemplate tests getTkgVersionBundleFromVAppTemplate function func Test_getTkgVersionBundleFromVAppTemplate(t *testing.T) { tests := []struct { From ad16187904677c7d73c6153648d2c11dc79cdf54 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 27 Feb 2024 17:10:22 +0100 Subject: [PATCH 099/115] Add unit tests for getCseComponentsVersions Signed-off-by: abarreiro --- govcd/cse_util.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 371687956..82958eaf9 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -21,9 +21,10 @@ func getCseComponentsVersions(cseVersion semver.Version) (*cseComponentsVersions v43, _ := semver.NewVersion("4.3.0") v42, _ := semver.NewVersion("4.2.0") v41, _ := semver.NewVersion("4.1.0") + err := fmt.Errorf("the Container Service Extension version '%s' is not supported", cseVersion.String()) if cseVersion.GreaterThanOrEqual(v43) { - return nil, fmt.Errorf("the Container Service Extension version '%s' is not supported", cseVersion.String()) + return nil, err } if cseVersion.GreaterThanOrEqual(v42) { return &cseComponentsVersions{ @@ -39,7 +40,7 @@ func getCseComponentsVersions(cseVersion semver.Version) (*cseComponentsVersions CseInterfaceVersion: "1.0.0", }, nil } - return nil, fmt.Errorf("the Container Service Extension version '%s' is not supported", cseVersion.String()) + return nil, err } // cseConvertToCseKubernetesClusterType takes a generic RDE that must represent an existing CSE Kubernetes cluster, From 58edf9a6858bf89bca69465617cfdf2170cc4ede Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 28 Feb 2024 09:35:08 +0100 Subject: [PATCH 100/115] Remove API Token from YAML Signed-off-by: abarreiro --- govcd/cse/4.1/capiyaml_cluster.tmpl | 10 ---------- govcd/cse/4.2.0/capiyaml_cluster.tmpl | 10 ---------- govcd/cse_internal.go | 3 --- 3 files changed, 23 deletions(-) diff --git a/govcd/cse/4.1/capiyaml_cluster.tmpl b/govcd/cse/4.1/capiyaml_cluster.tmpl index 92a6bb538..a4b225c0d 100644 --- a/govcd/cse/4.1/capiyaml_cluster.tmpl +++ b/govcd/cse/4.1/capiyaml_cluster.tmpl @@ -30,16 +30,6 @@ spec: name: "{{.ClusterName}}" namespace: "{{.TargetNamespace}}" --- -apiVersion: v1 -kind: Secret -metadata: - name: capi-user-credentials - namespace: {{.TargetNamespace}} -type: Opaque -data: - username: "{{.UsernameB64}}" - refreshToken: "{{.ApiTokenB64}}" ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: VCDCluster metadata: diff --git a/govcd/cse/4.2.0/capiyaml_cluster.tmpl b/govcd/cse/4.2.0/capiyaml_cluster.tmpl index 92a6bb538..a4b225c0d 100644 --- a/govcd/cse/4.2.0/capiyaml_cluster.tmpl +++ b/govcd/cse/4.2.0/capiyaml_cluster.tmpl @@ -30,16 +30,6 @@ spec: name: "{{.ClusterName}}" namespace: "{{.TargetNamespace}}" --- -apiVersion: v1 -kind: Secret -metadata: - name: capi-user-credentials - namespace: {{.TargetNamespace}} -type: Opaque -data: - username: "{{.UsernameB64}}" - refreshToken: "{{.ApiTokenB64}}" ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: VCDCluster metadata: diff --git a/govcd/cse_internal.go b/govcd/cse_internal.go index c60ef9a08..0adfd83d3 100644 --- a/govcd/cse_internal.go +++ b/govcd/cse_internal.go @@ -3,7 +3,6 @@ package govcd import ( "bytes" "embed" - "encoding/base64" "encoding/json" "fmt" "strconv" @@ -98,8 +97,6 @@ func (clusterSettings *cseClusterSettingsInternal) generateCapiYamlAsJsonString( "TargetNamespace": clusterSettings.Name + "-ns", "TkrVersion": clusterSettings.TkgVersionBundle.TkrVersion, "TkgVersion": clusterSettings.TkgVersionBundle.TkgVersion, - "UsernameB64": base64.StdEncoding.EncodeToString([]byte(clusterSettings.Owner)), - "ApiTokenB64": base64.StdEncoding.EncodeToString([]byte(clusterSettings.ApiToken)), "PodCidr": clusterSettings.PodCidr, "ServiceCidr": clusterSettings.ServiceCidr, "VcdSite": clusterSettings.VcdUrl, From 21284a990aeee285af9131fbe0b1432c16ba347d Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 28 Feb 2024 10:24:19 +0100 Subject: [PATCH 101/115] Address some suggestions Signed-off-by: abarreiro --- govcd/cse.go | 12 ++++++++---- govcd/cse_internal_unit_test.go | 6 +++++- govcd/cse_type.go | 6 ++++++ govcd/cse_util.go | 10 +++++++--- govcd/cse_util_unit_test.go | 10 ++++++++++ govcd/system.go | 2 +- govcd/system_test.go | 13 +++++++++++++ 7 files changed, 50 insertions(+), 9 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index b3534aa8a..7c1375c6d 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -56,7 +56,7 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterSettings CseClusterSettin return "", err } - rde, err := createRdeAndGetFromTask(org.client, "vmware", "capvcdCluster", cseSubcomponents.CapvcdRdeTypeVersion, + rde, err := createRdeAndGetFromTask(org.client, cseKubernetesClusterVendor, cseKubernetesClusterNamespace, cseSubcomponents.CapvcdRdeTypeVersion, types.DefinedEntity{ EntityType: internalSettings.RdeType.ID, Name: internalSettings.Name, @@ -85,7 +85,7 @@ func (org *Org) CseGetKubernetesClustersByName(cseVersion semver.Version, name s return nil, err } - rdes, err := getRdesByName(org.client, "vmware", "capvcdCluster", cseSubcomponents.CapvcdRdeTypeVersion, name) + rdes, err := getRdesByName(org.client, cseKubernetesClusterVendor, cseKubernetesClusterNamespace, cseSubcomponents.CapvcdRdeTypeVersion, name) if err != nil { return nil, err } @@ -132,8 +132,12 @@ func (cluster *CseKubernetesCluster) GetKubeconfig(refresh bool) (string, error) } } + if cluster.State == "" { + return "", fmt.Errorf("cannot get a Kubeconfig of a Kubernetes cluster that does not have a state (expected 'provisioned')") + } + if cluster.State != "provisioned" { - return "", fmt.Errorf("cannot get a Kubeconfig of a Kubernetes cluster that is not in 'provisioned' state") + return "", fmt.Errorf("cannot get a Kubeconfig of a Kubernetes cluster that is not in 'provisioned' state. It is '%s'", cluster.State) } rde, err := getRdeById(cluster.client, cluster.ID) @@ -276,7 +280,7 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh // automatically by the CSE Server. v411, err := semver.NewVersion("4.1.1") if err != nil { - return fmt.Errorf("can't update the 'Auto Repair on Errors' flag: %s", err) + return err } if cluster.CseVersion.GreaterThanOrEqual(v411) { return fmt.Errorf("the 'Auto Repair on Errors' flag can't be changed after the cluster is created since CSE 4.1.1") diff --git a/govcd/cse_internal_unit_test.go b/govcd/cse_internal_unit_test.go index 5b1638008..3a9954958 100644 --- a/govcd/cse_internal_unit_test.go +++ b/govcd/cse_internal_unit_test.go @@ -10,7 +10,9 @@ import ( "testing" ) -// Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString +// Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString tests the generateCapiYamlAsJsonString method with a +// cseClusterSettingsInternal receiver. Given some valid or invalid CSE Settings, the tests runs the generateCapiYamlAsJsonString +// method and checks that the returned JSON string corresponds to the expected settings that were specified. func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { cseVersion, err := semver.NewVersion("4.2.0") if err != nil { @@ -121,6 +123,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) PodCidr: "100.96.0.0/11", ServiceCidr: "100.64.0.0/13", }, + // The expected result is the base YAML without the MachineHealthCheck expectedFunc: func() []map[string]interface{} { var result []map[string]interface{} for _, doc := range baseUnmarshaledYaml { @@ -179,6 +182,7 @@ func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) PodCidr: "100.96.0.0/11", ServiceCidr: "100.64.0.0/13", }, + // The expected result is the base YAML with the Control Plane extra IPs expectedFunc: func() []map[string]interface{} { var result []map[string]interface{} for _, doc := range baseUnmarshaledYaml { diff --git a/govcd/cse_type.go b/govcd/cse_type.go index db07c9cd7..a25a2736a 100644 --- a/govcd/cse_type.go +++ b/govcd/cse_type.go @@ -194,3 +194,9 @@ type cseComponentsVersions struct { CapvcdRdeTypeVersion string CseInterfaceVersion string } + +// Constants that define the RDE Type of a CSE Kubernetes cluster +const ( + cseKubernetesClusterVendor = "vmware" + cseKubernetesClusterNamespace = "capvcdCluster" +) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 82958eaf9..fdf00f7fb 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -48,10 +48,14 @@ func getCseComponentsVersions(cseVersion semver.Version) (*cseComponentsVersions // it is easy to explore and consume. If the input RDE is not a CSE Kubernetes cluster, this method // will obviously return an error. // +// The transformation from a generic RDE to a CseKubernetesCluster is done by querying VCD for every needed item, +// such as Network IDs, Compute Policies IDs, vApp Template IDs, etc. It deeply explores the RDE contents +// (even the CAPI YAML) to retrieve information and getting the missing pieces from VCD. +// // WARNING: Don't use this method inside loops or avoid calling it multiple times in a row, as it performs many queries // to VCD. func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesCluster, error) { - requiredType := "vmware:capvcdCluster" + requiredType := fmt.Sprintf("%s:%s", cseKubernetesClusterVendor, cseKubernetesClusterNamespace) if !strings.Contains(rde.DefinedEntity.ID, requiredType) || !strings.Contains(rde.DefinedEntity.EntityType, requiredType) { return nil, fmt.Errorf("the receiver RDE is not a '%s' entity, it is '%s'", requiredType, rde.DefinedEntity.EntityType) @@ -71,7 +75,7 @@ func cseConvertToCseKubernetesClusterType(rde *DefinedEntity) (*CseKubernetesClu result := &CseKubernetesCluster{ CseClusterSettings: CseClusterSettings{ Name: rde.DefinedEntity.Name, - ApiToken: "******", // We can't return this one, we return the "standard" 6-asterisk value + ApiToken: "******", // We must not return this one, we return the "standard" 6-asterisk value AutoRepairOnErrors: capvcd.Spec.VcdKe.AutoRepairOnErrors, ControlPlane: CseControlPlaneSettings{}, }, @@ -651,7 +655,7 @@ func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClus if err != nil { return nil, err } - rdeType, err := getRdeType(org.client, "vmware", "capvcdCluster", cseComponentsVersions.CapvcdRdeTypeVersion) + rdeType, err := getRdeType(org.client, cseKubernetesClusterVendor, cseKubernetesClusterNamespace, cseComponentsVersions.CapvcdRdeTypeVersion) if err != nil { return nil, err } diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index 3699a7fbc..d63833cd4 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -45,6 +45,16 @@ func Test_getCseComponentsVersions(t *testing.T) { }, wantErr: false, }, + { + name: "CSE 4.1.1a is equivalent to 4.1.1", + cseVersion: "4.1.1a", + want: &cseComponentsVersions{ + VcdKeConfigRdeTypeVersion: "1.1.0", + CapvcdRdeTypeVersion: "1.2.0", + CseInterfaceVersion: "1.0.0", + }, + wantErr: false, + }, { name: "CSE 4.2 is supported", cseVersion: "4.2", diff --git a/govcd/system.go b/govcd/system.go index 14fee5b90..6bd93afc7 100644 --- a/govcd/system.go +++ b/govcd/system.go @@ -1192,7 +1192,7 @@ func queryAdminOrgVdcStorageProfilesByVdcId(client *Client, vdcId string) ([]*ty // queryOrgVdcStorageProfilesByVdcId finds all Storage Profiles of a VDC func queryOrgVdcStorageProfilesByVdcId(client *Client, vdcId string) ([]*types.QueryResultOrgVdcStorageProfileRecordType, error) { if client.IsSysAdmin { - return nil, errors.New("can't query type QueryResultOrgVdcStorageProfileRecordType as System administrator") + return nil, errors.New("can't query type QtOrgVdcStorageProfile as System administrator") } results, err := client.QueryWithNotEncodedParams(nil, map[string]string{ "type": types.QtOrgVdcStorageProfile, diff --git a/govcd/system_test.go b/govcd/system_test.go index acf237da9..14c39bbe2 100644 --- a/govcd/system_test.go +++ b/govcd/system_test.go @@ -649,6 +649,19 @@ func (vcd *TestVCD) Test_QueryOrgVdcStorageProfileByID(check *C) { } check.Assert(storageProfileFound, Equals, true) + + vdcStorageProfiles, err := queryOrgVdcStorageProfilesByVdcId(&vcd.client.Client, vcd.vdc.Vdc.ID) + check.Assert(err, IsNil) + storageProfileFound = false + for _, profile := range vdcStorageProfiles { + id, err := GetUuidFromHref(profile.HREF, true) + check.Assert(err, IsNil) + if id == storageProfileID { + storageProfileFound = true + break + } + } + check.Assert(storageProfileFound, Equals, true) } func (vcd *TestVCD) Test_QueryNetworkPoolByName(check *C) { From 8b1a5fa059e87ca0e87f1549ac51cfd07c4141f9 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 28 Feb 2024 12:07:30 +0100 Subject: [PATCH 102/115] Apply suggestion Signed-off-by: abarreiro --- govcd/cse.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 7c1375c6d..399a84240 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -304,8 +304,9 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh return err } - // We do this loop to increase the chances that the Kubernetes cluster is successfully updated, as this method will be - // "fighting" with the CSE Server + // We do this loop to increase the chances that the Kubernetes cluster is successfully updated, as the update operation + // can clash with the CSE Server updates on the same RDE. If the CSE Server does an update just before we do, the ETag + // verification will fail, so we must retry. retries := 0 maxRetries := 5 updated := false From d6820a6a22d3f94793c513fc7b4cd9a8952b5b05 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 28 Feb 2024 12:22:14 +0100 Subject: [PATCH 103/115] Fix unit tests Signed-off-by: abarreiro --- govcd/cse_yaml_unit_test.go | 2 +- govcd/test-resources/capiYaml.yaml | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index 1793e8811..bc6f9fb6e 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -336,7 +336,7 @@ func Test_unmarshalMultplieYamlDocuments(t *testing.T) { { name: "unmarshal correct amount of documents", yamlDocuments: string(capiYaml), - want: 9, + want: 8, wantErr: false, }, { diff --git a/govcd/test-resources/capiYaml.yaml b/govcd/test-resources/capiYaml.yaml index 762a8a690..79bc707e7 100644 --- a/govcd/test-resources/capiYaml.yaml +++ b/govcd/test-resources/capiYaml.yaml @@ -96,16 +96,6 @@ spec: name: "test1" namespace: "test1-ns" --- -apiVersion: v1 -kind: Secret -metadata: - name: capi-user-credentials - namespace: test1-ns -type: Opaque -data: - username: "ZHVtbXk=" - refreshToken: "ZHVtbXk=" ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: VCDCluster metadata: From 02609cc2fb05a10bf436b5ad4542cbb7e4fe31e3 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 28 Feb 2024 15:56:26 +0100 Subject: [PATCH 104/115] Remove comment Signed-off-by: abarreiro --- govcd/cse_yaml.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index fcb7e87f1..12e2566bd 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -318,8 +318,6 @@ func cseUpdateNodeHealthCheckInYaml(yamlDocuments []map[string]interface{}, clus // There is a MachineHealthCheck block if machineHealthCheckEnabled { // We want it, but it is already there, so nothing to do - // TODO: What happens in UI if the VCDKEConfig MHC values are changed, does it get reflected in the cluster? - // If that's the case, we might need to update this value always return result, nil } From 80deaea68d1b3f48c8daf7d471afb456392998e5 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 28 Feb 2024 16:33:54 +0100 Subject: [PATCH 105/115] Change t.Fatalf to t.Fatal on trivial returns Signed-off-by: abarreiro --- govcd/cse_internal_unit_test.go | 2 +- govcd/cse_util_unit_test.go | 8 ++++---- govcd/cse_yaml_unit_test.go | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/govcd/cse_internal_unit_test.go b/govcd/cse_internal_unit_test.go index 3a9954958..32029bc68 100644 --- a/govcd/cse_internal_unit_test.go +++ b/govcd/cse_internal_unit_test.go @@ -16,7 +16,7 @@ import ( func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { cseVersion, err := semver.NewVersion("4.2.0") if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } capiYaml, err := os.ReadFile("test-resources/capiYaml.yaml") if err != nil { diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index d63833cd4..5fbc3a0ed 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -321,15 +321,15 @@ func Test_getCseTemplate(t *testing.T) { tmpl41, err := getCseTemplate(*v41, "rde") if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } tmpl410, err := getCseTemplate(*v410, "rde") if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } tmpl411, err := getCseTemplate(*v411, "rde") if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } if tmpl41 == "" || tmpl41 != tmpl410 || tmpl41 != tmpl411 || tmpl410 != tmpl411 { t.Fatalf("templates should be the same:\n4.1: %s\n4.1.0: %s\n4.1.1: %s", tmpl41, tmpl410, tmpl411) @@ -337,7 +337,7 @@ func Test_getCseTemplate(t *testing.T) { tmpl420, err := getCseTemplate(*v420, "rde") if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } if tmpl420 == "" { t.Fatalf("the obtained template for %s is empty", v420.String()) diff --git a/govcd/cse_yaml_unit_test.go b/govcd/cse_yaml_unit_test.go index bc6f9fb6e..e4258c273 100644 --- a/govcd/cse_yaml_unit_test.go +++ b/govcd/cse_yaml_unit_test.go @@ -44,7 +44,7 @@ func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { }}, }) if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } // We call the function to update the old OVA with the new one @@ -66,12 +66,12 @@ func Test_cseUpdateKubernetesTemplateInYaml(t *testing.T) { } newTkgBundle, err := getTkgVersionBundleFromVAppTemplate(newOva) if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } err = cseUpdateKubernetesTemplateInYaml(yamlDocs, newOva) if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } updatedYaml, err := marshalMultipleYamlDocuments(yamlDocs) @@ -141,7 +141,7 @@ func Test_cseUpdateWorkerPoolsInYaml(t *testing.T) { } err = cseUpdateWorkerPoolsInYaml(yamlDocs, newNodePools) if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } // The worker pools should have now the new details updated @@ -212,7 +212,7 @@ func Test_cseUpdateControlPlaneInYaml(t *testing.T) { } err = cseUpdateControlPlaneInYaml(yamlDocs, newControlPlane) if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } // The control plane should have now the new details updated @@ -278,7 +278,7 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { // Deactivates Machine Health Check yamlDocs, err = cseUpdateNodeHealthCheckInYaml(yamlDocs, clusterName, *v, vcdKeConfig{}) if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } // The resulting documents should not have that document @@ -296,7 +296,7 @@ func Test_cseUpdateNodeHealthCheckInYaml(t *testing.T) { NodeUnknownTimeout: "78", }) if err != nil { - t.Fatalf("%s", err) + t.Fatal(err) } // The resulting documents should have a MachineHealthCheck From 6a9e1c3254d0038583882e5a8d083bb3bc3fc073 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 28 Feb 2024 16:44:17 +0100 Subject: [PATCH 106/115] Set cluster creation timeout in test Signed-off-by: abarreiro --- govcd/cse_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index bea1b68ec..a96769796 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -26,7 +26,7 @@ func requireCseConfig(check *C, testConfig TestConfig) { cseConfigType := cseConfigValues.Type() for i := 0; i < cseConfigValues.NumField(); i++ { if cseConfigValues.Field(i).String() == "" { - check.Skip(fmt.Sprintf("%s the config value '%s' inside 'cse' block of govcd_test_config.yaml is not set", skippedPrefix, strings.ToLower(cseConfigType.Field(i).Name))) + check.Skip(fmt.Sprintf("%s the config value '%s' inside 'cse' block of govcd_test_config.yaml is not set", skippedPrefix, cseConfigType.Field(i).Name)) } } } @@ -113,7 +113,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { ServiceCidr: "100.64.0.0/13", AutoRepairOnErrors: true, } - cluster, err := org.CseCreateKubernetesCluster(clusterSettings, 0) + cluster, err := org.CseCreateKubernetesCluster(clusterSettings, 150) // We assure that the cluster gets always deleted, even if the creation failed. // Deletion process only needs the cluster ID From c2bcfe8b712f8c064203def41e749f7bb3a81ec4 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 28 Feb 2024 16:53:39 +0100 Subject: [PATCH 107/115] Fix previous commit/timeout Signed-off-by: abarreiro --- govcd/cse_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/govcd/cse_test.go b/govcd/cse_test.go index a96769796..498e1054c 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -15,6 +15,7 @@ import ( "os" "reflect" "strings" + "time" ) func requireCseConfig(check *C, testConfig TestConfig) { @@ -113,7 +114,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { ServiceCidr: "100.64.0.0/13", AutoRepairOnErrors: true, } - cluster, err := org.CseCreateKubernetesCluster(clusterSettings, 150) + cluster, err := org.CseCreateKubernetesCluster(clusterSettings, 150*time.Minute) // We assure that the cluster gets always deleted, even if the creation failed. // Deletion process only needs the cluster ID From 5018bed64835ea8891e54ccb464ea1fa37804738 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 1 Mar 2024 12:20:16 +0100 Subject: [PATCH 108/115] Fixes and add a new test Signed-off-by: abarreiro --- govcd/cse.go | 9 ++- govcd/cse_test.go | 139 ++++++++++++++++++++++++++++++++++++++++++++++ govcd/cse_util.go | 22 +++++++- 3 files changed, 168 insertions(+), 2 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 399a84240..b1c8635fe 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -27,7 +27,10 @@ func (org *Org) CseCreateKubernetesCluster(clusterData CseClusterSettings, timeo err = waitUntilClusterIsProvisioned(org.client, clusterId, timeout) if err != nil { - return &CseKubernetesCluster{ID: clusterId}, err + return &CseKubernetesCluster{ + client: org.client, + ID: clusterId, + }, err } return getCseKubernetesClusterById(org.client, clusterId) @@ -205,6 +208,10 @@ func (cluster *CseKubernetesCluster) GetSupportedUpgrades(refreshOvas bool) ([]* if refreshOvas { cluster.supportedUpgrades = make([]*types.VAppTemplate, 0) } + if cluster.State != "provisioned" { + cluster.supportedUpgrades = make([]*types.VAppTemplate, 0) + return cluster.supportedUpgrades, nil + } if len(cluster.supportedUpgrades) > 0 { return cluster.supportedUpgrades, nil } diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 498e1054c..7957dfff3 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -120,6 +120,7 @@ func (vcd *TestVCD) Test_Cse(check *C) { // Deletion process only needs the cluster ID defer func() { check.Assert(cluster, NotNil) + check.Assert(cluster.client, NotNil) check.Assert(cluster.ID, Not(Equals), "") err = cluster.Delete(0) check.Assert(err, IsNil) @@ -240,6 +241,144 @@ func (vcd *TestVCD) Test_Cse(check *C) { } } +// Test_Cse_Failure tests cluster creation errors and their consequences +func (vcd *TestVCD) Test_Cse_Failure(check *C) { + requireCseConfig(check, vcd.config) + + // Prerequisites: We need to read several items before creating the cluster. + org, err := vcd.client.GetOrgByName(vcd.config.Cse.TenantOrg) + check.Assert(err, IsNil) + + catalog, err := org.GetCatalogByName(vcd.config.Cse.OvaCatalog, false) + check.Assert(err, IsNil) + + ova, err := catalog.GetVAppTemplateByName(vcd.config.Cse.OvaName) + check.Assert(err, IsNil) + + vdc, err := org.GetVDCByName(vcd.config.Cse.TenantVdc, false) + check.Assert(err, IsNil) + + net, err := vdc.GetOrgVdcNetworkByName(vcd.config.Cse.RoutedNetwork, false) + check.Assert(err, IsNil) + + sp, err := vdc.FindStorageProfileReference(vcd.config.Cse.StorageProfile) + check.Assert(err, IsNil) + + policies, err := vcd.client.GetAllVdcComputePoliciesV2(url.Values{ + "filter": []string{"name==TKG small"}, + }) + check.Assert(err, IsNil) + check.Assert(len(policies), Equals, 1) + + token, err := vcd.client.CreateToken(vcd.config.Provider.SysOrg, check.TestName()) + check.Assert(err, IsNil) + defer func() { + err = token.Delete() + check.Assert(err, IsNil) + }() + AddToCleanupListOpenApi(token.Token.Name, check.TestName(), types.OpenApiPathVersion1_0_0+types.OpenApiEndpointTokens+token.Token.ID) + + apiToken, err := token.GetInitialApiToken() + check.Assert(err, IsNil) + + cseVersion, err := semver.NewVersion(vcd.config.Cse.Version) + check.Assert(err, IsNil) + check.Assert(cseVersion, NotNil) + + componentsVersions, err := getCseComponentsVersions(*cseVersion) + check.Assert(err, IsNil) + check.Assert(componentsVersions, NotNil) + + // Create the cluster + clusterSettings := CseClusterSettings{ + Name: "test-cse-fail", + OrganizationId: org.Org.ID, + VdcId: vdc.Vdc.ID, + NetworkId: net.OrgVDCNetwork.ID, + KubernetesTemplateOvaId: ova.VAppTemplate.ID, + CseVersion: *cseVersion, + ControlPlane: CseControlPlaneSettings{ + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyId: policies[0].VdcComputePolicyV2.ID, + StorageProfileId: sp.ID, + Ip: "", + }, + WorkerPools: []CseWorkerPoolSettings{{ + Name: "worker-pool-1", + MachineCount: 1, + DiskSizeGi: 20, + SizingPolicyId: policies[0].VdcComputePolicyV2.ID, + StorageProfileId: sp.ID, + }}, + Owner: vcd.config.Provider.User, + ApiToken: apiToken.RefreshToken, + NodeHealthCheck: true, + PodCidr: "1.1.1.1/24", // This should make the cluster fail + ServiceCidr: "1.1.1.1/24", // This should make the cluster fail + AutoRepairOnErrors: false, // Must be false to avoid never-ending loops + } + cluster, err := org.CseCreateKubernetesCluster(clusterSettings, 150*time.Minute) + + // We assure that the cluster gets always deleted. + // Deletion process only needs the cluster ID + defer func() { + check.Assert(cluster, NotNil) + check.Assert(cluster.client, NotNil) + check.Assert(cluster.ID, Not(Equals), "") + err = cluster.Delete(0) + check.Assert(err, IsNil) + }() + + check.Assert(err, NotNil) + check.Assert(cluster.client, NotNil) + check.Assert(cluster.ID, Not(Equals), "") + + clusterGet, err := vcd.client.CseGetKubernetesClusterById(cluster.ID) + check.Assert(err, IsNil) + // We don't get an error when we retrieve a failed cluster, but some fields are missing + check.Assert(clusterGet.ID, Equals, cluster.ID) + check.Assert(clusterGet.Etag, Not(Equals), "") + check.Assert(clusterGet.State, Equals, "error") + check.Assert(len(clusterGet.Events), Not(Equals), 0) + + err = cluster.Refresh() + check.Assert(err, IsNil) + assertCseClusterEquals(check, cluster, clusterGet) + + allClusters, err := org.CseGetKubernetesClustersByName(clusterGet.CseVersion, clusterGet.Name) + check.Assert(err, IsNil) + check.Assert(len(allClusters), Equals, 1) + assertCseClusterEquals(check, allClusters[0], clusterGet) + check.Assert(allClusters[0].Etag, Equals, "") // Can't recover ETag by name + + _, err = cluster.GetKubeconfig(false) + check.Assert(err, NotNil) + + // All updates should fail + err = cluster.UpdateWorkerPools(map[string]CseWorkerPoolUpdateInput{clusterSettings.WorkerPools[0].Name: {MachineCount: 1}}, true) + check.Assert(err, NotNil) + err = cluster.AddWorkerPools([]CseWorkerPoolSettings{{ + Name: "i-dont-care-i-will-fail", + MachineCount: 1, + DiskSizeGi: 20, + }}, true) + check.Assert(err, NotNil) + err = cluster.UpdateControlPlane(CseControlPlaneUpdateInput{MachineCount: 1}, true) + check.Assert(err, NotNil) + err = cluster.SetNodeHealthCheck(false, true) + check.Assert(err, NotNil) + err = cluster.SetAutoRepairOnErrors(false, true) + check.Assert(err, NotNil) + + upgradeOvas, err := cluster.GetSupportedUpgrades(true) + check.Assert(err, IsNil) + check.Assert(len(upgradeOvas), Equals, 0) + + err = cluster.UpgradeCluster(clusterSettings.KubernetesTemplateOvaId, true) + check.Assert(err, NotNil) +} + func assertCseClusterCreation(check *C, createdCluster *CseKubernetesCluster, settings CseClusterSettings, expectedKubernetesData tkgVersionBundle) { check.Assert(createdCluster, NotNil) check.Assert(createdCluster.CseVersion.Original(), Equals, settings.CseVersion.Original()) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index fdf00f7fb..6c7d244e3 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -7,6 +7,7 @@ import ( semver "github.com/hashicorp/go-version" "github.com/vmware/go-vcloud-director/v2/types/v56" "github.com/vmware/go-vcloud-director/v2/util" + "net" "net/url" "regexp" "sort" @@ -597,9 +598,25 @@ func (input *CseClusterSettings) validate() error { if input.PodCidr == "" { return fmt.Errorf("the Pod CIDR is required") } + if _, _, err := net.ParseCIDR(input.PodCidr); err != nil { + return fmt.Errorf("the Pod CIDR is malformed: %s", err) + } if input.ServiceCidr == "" { return fmt.Errorf("the Service CIDR is required") } + if _, _, err := net.ParseCIDR(input.ServiceCidr); err != nil { + return fmt.Errorf("the Service CIDR is malformed: %s", err) + } + if input.VirtualIpSubnet != "" { + if _, _, err := net.ParseCIDR(input.VirtualIpSubnet); err != nil { + return fmt.Errorf("the Virtual IP Subnet is malformed: %s", err) + } + } + if input.ControlPlane.Ip != "" { + if r := net.ParseIP(input.ControlPlane.Ip); r == nil { + return fmt.Errorf("the Control Plane IP is malformed: %s", input.ControlPlane.Ip) + } + } return nil } @@ -669,7 +686,10 @@ func (input *CseClusterSettings) toCseClusterSettingsInternal(org Org) (*cseClus storageProfileIds = append(storageProfileIds, w.StorageProfileId) } computePolicyIds = append(computePolicyIds, input.ControlPlane.SizingPolicyId, input.ControlPlane.PlacementPolicyId) - storageProfileIds = append(storageProfileIds, input.ControlPlane.StorageProfileId, input.DefaultStorageClass.StorageProfileId) + storageProfileIds = append(storageProfileIds, input.ControlPlane.StorageProfileId) + if input.DefaultStorageClass != nil { + storageProfileIds = append(storageProfileIds, input.DefaultStorageClass.StorageProfileId) + } idToNameCache, err := idToNames(org.client, computePolicyIds, storageProfileIds) if err != nil { From d1f974776d6b4867173914dcd99d353edcd74099 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 4 Mar 2024 12:28:13 +0100 Subject: [PATCH 109/115] Revert YAML library changes Signed-off-by: abarreiro --- go.mod | 1 + go.sum | 2 ++ govcd/api_vcd_test.go | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9ae8e3a92..4c593455f 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/kr/pretty v0.2.1 github.com/peterhellberg/link v1.1.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 + gopkg.in/yaml.v2 v2.4.0 sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 8b4be2b5e..2cf86b169 100644 --- a/go.sum +++ b/go.sum @@ -26,5 +26,7 @@ golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/govcd/api_vcd_test.go b/govcd/api_vcd_test.go index 771fe0e2f..3f1238e08 100644 --- a/govcd/api_vcd_test.go +++ b/govcd/api_vcd_test.go @@ -11,13 +11,13 @@ import ( "encoding/json" "flag" "fmt" + "gopkg.in/yaml.v2" "net/http" "net/url" "os" "path/filepath" "regexp" "runtime" - "sigs.k8s.io/yaml" "strings" "sync" "testing" From 39b8d8535d91019fd4e418bad0bba60e1bb2ba52 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Mon, 4 Mar 2024 14:57:14 +0100 Subject: [PATCH 110/115] Fix replace Signed-off-by: abarreiro --- go.mod | 5 ++++- go.sum | 6 ++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 4c593455f..c75e4b531 100644 --- a/go.mod +++ b/go.mod @@ -20,4 +20,7 @@ require ( golang.org/x/text v0.14.0 ) -replace gopkg.in/check.v1 => github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c +replace ( + gopkg.in/check.v1 => github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c + gopkg.in/yaml.v2 => github.com/go-yaml/yaml/v2 v2.2.2 +) diff --git a/go.sum b/go.sum index 2cf86b169..27859793b 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c h1:3LdnoQiW6yLkxRIwSU3pbYp3zqW1daDgoOcOD09OzJs= github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +github.com/go-yaml/yaml/v2 v2.2.2 h1:uw2m9KuKRscWGAkuyoBGQcZSdibhmuXKSJ3+9Tj3zXc= +github.com/go-yaml/yaml/v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= @@ -24,9 +26,5 @@ golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRj golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= From 81c3a6043456177451e6170343d56f1adfde334a Mon Sep 17 00:00:00 2001 From: abarreiro Date: Tue, 5 Mar 2024 17:24:14 +0100 Subject: [PATCH 111/115] Revert changes by suggestion Signed-off-by: abarreiro --- govcd/system.go | 2 +- govcd/system_test.go | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/govcd/system.go b/govcd/system.go index 6bd93afc7..532d64d44 100644 --- a/govcd/system.go +++ b/govcd/system.go @@ -1192,7 +1192,7 @@ func queryAdminOrgVdcStorageProfilesByVdcId(client *Client, vdcId string) ([]*ty // queryOrgVdcStorageProfilesByVdcId finds all Storage Profiles of a VDC func queryOrgVdcStorageProfilesByVdcId(client *Client, vdcId string) ([]*types.QueryResultOrgVdcStorageProfileRecordType, error) { if client.IsSysAdmin { - return nil, errors.New("can't query type QtOrgVdcStorageProfile as System administrator") + return nil, errors.New("can't query type QueryResultAdminOrgVdcStorageProfileRecordType as System administrator") } results, err := client.QueryWithNotEncodedParams(nil, map[string]string{ "type": types.QtOrgVdcStorageProfile, diff --git a/govcd/system_test.go b/govcd/system_test.go index 14c39bbe2..acf237da9 100644 --- a/govcd/system_test.go +++ b/govcd/system_test.go @@ -649,19 +649,6 @@ func (vcd *TestVCD) Test_QueryOrgVdcStorageProfileByID(check *C) { } check.Assert(storageProfileFound, Equals, true) - - vdcStorageProfiles, err := queryOrgVdcStorageProfilesByVdcId(&vcd.client.Client, vcd.vdc.Vdc.ID) - check.Assert(err, IsNil) - storageProfileFound = false - for _, profile := range vdcStorageProfiles { - id, err := GetUuidFromHref(profile.HREF, true) - check.Assert(err, IsNil) - if id == storageProfileID { - storageProfileFound = true - break - } - } - check.Assert(storageProfileFound, Equals, true) } func (vcd *TestVCD) Test_QueryNetworkPoolByName(check *C) { From 7b022788a3fe716e5c853cc58c5b8c4b54e51d17 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 6 Mar 2024 11:23:33 +0100 Subject: [PATCH 112/115] Fix dumb AutoRepairOnErrors error message when settings dont change Signed-off-by: abarreiro --- govcd/cse.go | 3 ++- govcd/cse_test.go | 9 ++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index b1c8635fe..454f99881 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -282,9 +282,10 @@ func (cluster *CseKubernetesCluster) Update(input CseClusterUpdateInput, refresh return fmt.Errorf("can't update a Kubernetes cluster that is not in 'provisioned' state, as it is in '%s'", cluster.capvcdType.Status.VcdKe.State) } - if input.AutoRepairOnErrors != nil { + if input.AutoRepairOnErrors != nil && *input.AutoRepairOnErrors != cluster.AutoRepairOnErrors { // Since CSE 4.1.1, the AutoRepairOnError toggle can't be modified and is turned off // automatically by the CSE Server. + v411, err := semver.NewVersion("4.1.1") if err != nil { return err diff --git a/govcd/cse_test.go b/govcd/cse_test.go index 7957dfff3..3f09742b3 100644 --- a/govcd/cse_test.go +++ b/govcd/cse_test.go @@ -192,15 +192,10 @@ func (vcd *TestVCD) Test_Cse(check *C) { check.Assert(cluster.NodeHealthCheck, Equals, false) // Update the auto repair flag - v411, err := semver.NewVersion("4.1.1") check.Assert(err, IsNil) err = cluster.SetAutoRepairOnErrors(false, true) - if cluster.CseVersion.GreaterThan(v411) { - check.Assert(err, NotNil) // Can't be changed since CSE 4.1.1 - } else { - check.Assert(err, IsNil) - } - check.Assert(cluster.NodeHealthCheck, Equals, false) + check.Assert(err, IsNil) // It won't fail in CSE >4.1.0 as the flag is already false, so we update nothing. + check.Assert(cluster.AutoRepairOnErrors, Equals, false) // Upgrade the cluster if possible upgradeOvas, err := cluster.GetSupportedUpgrades(true) From bf94572f64614ddc92ebf321434ddced8e3c43fc Mon Sep 17 00:00:00 2001 From: abarreiro Date: Wed, 6 Mar 2024 12:55:09 +0100 Subject: [PATCH 113/115] Add 4.2.1 Signed-off-by: abarreiro --- .changes/v2.23.0/645-features.md | 3 ++- govcd/cse/{4.2.0 => 4.2}/capiyaml_cluster.tmpl | 0 govcd/cse/{4.2.0 => 4.2}/capiyaml_mhc.tmpl | 0 govcd/cse/{4.2.0 => 4.2}/capiyaml_workerpool.tmpl | 0 govcd/cse/{4.2.0 => 4.2}/rde.tmpl | 0 govcd/cse_internal_unit_test.go | 2 +- govcd/cse_util_unit_test.go | 8 ++++---- 7 files changed, 7 insertions(+), 6 deletions(-) rename govcd/cse/{4.2.0 => 4.2}/capiyaml_cluster.tmpl (100%) rename govcd/cse/{4.2.0 => 4.2}/capiyaml_mhc.tmpl (100%) rename govcd/cse/{4.2.0 => 4.2}/capiyaml_workerpool.tmpl (100%) rename govcd/cse/{4.2.0 => 4.2}/rde.tmpl (100%) diff --git a/.changes/v2.23.0/645-features.md b/.changes/v2.23.0/645-features.md index 38ed2dc6a..3b1c7c1c1 100644 --- a/.changes/v2.23.0/645-features.md +++ b/.changes/v2.23.0/645-features.md @@ -1,4 +1,5 @@ -* Added the type `CseKubernetesCluster` to manage Container Service Extension Kubernetes clusters [GH-645] +* Added the type `CseKubernetesCluster` to manage Container Service Extension Kubernetes clusters for versions 4.1.0, 4.1.1, + 4.2.0 and 4.2.1 [GH-645] * Added methods `Org.CseCreateKubernetesCluster` and `Org.CseCreateKubernetesClusterAsync` to create Kubernetes clusters in a VCD appliance with Container Service Extension installed [GH-645] * Added methods `VCDClient.CseGetKubernetesClusterById` and `Org.CseGetKubernetesClustersByName` to retrieve a diff --git a/govcd/cse/4.2.0/capiyaml_cluster.tmpl b/govcd/cse/4.2/capiyaml_cluster.tmpl similarity index 100% rename from govcd/cse/4.2.0/capiyaml_cluster.tmpl rename to govcd/cse/4.2/capiyaml_cluster.tmpl diff --git a/govcd/cse/4.2.0/capiyaml_mhc.tmpl b/govcd/cse/4.2/capiyaml_mhc.tmpl similarity index 100% rename from govcd/cse/4.2.0/capiyaml_mhc.tmpl rename to govcd/cse/4.2/capiyaml_mhc.tmpl diff --git a/govcd/cse/4.2.0/capiyaml_workerpool.tmpl b/govcd/cse/4.2/capiyaml_workerpool.tmpl similarity index 100% rename from govcd/cse/4.2.0/capiyaml_workerpool.tmpl rename to govcd/cse/4.2/capiyaml_workerpool.tmpl diff --git a/govcd/cse/4.2.0/rde.tmpl b/govcd/cse/4.2/rde.tmpl similarity index 100% rename from govcd/cse/4.2.0/rde.tmpl rename to govcd/cse/4.2/rde.tmpl diff --git a/govcd/cse_internal_unit_test.go b/govcd/cse_internal_unit_test.go index 32029bc68..3527854e9 100644 --- a/govcd/cse_internal_unit_test.go +++ b/govcd/cse_internal_unit_test.go @@ -14,7 +14,7 @@ import ( // cseClusterSettingsInternal receiver. Given some valid or invalid CSE Settings, the tests runs the generateCapiYamlAsJsonString // method and checks that the returned JSON string corresponds to the expected settings that were specified. func Test_cseClusterSettingsInternal_generateCapiYamlAsJsonString(t *testing.T) { - cseVersion, err := semver.NewVersion("4.2.0") + cseVersion, err := semver.NewVersion("4.2.1") if err != nil { t.Fatal(err) } diff --git a/govcd/cse_util_unit_test.go b/govcd/cse_util_unit_test.go index 5fbc3a0ed..5e98edb8e 100644 --- a/govcd/cse_util_unit_test.go +++ b/govcd/cse_util_unit_test.go @@ -314,9 +314,9 @@ func Test_getCseTemplate(t *testing.T) { if err != nil { t.Fatalf("could not create 4.1.1 version object") } - v420, err := semver.NewVersion("4.2.0") + v421, err := semver.NewVersion("4.2.1") if err != nil { - t.Fatalf("could not create 4.2.0 version object") + t.Fatalf("could not create 4.2.1 version object") } tmpl41, err := getCseTemplate(*v41, "rde") @@ -335,12 +335,12 @@ func Test_getCseTemplate(t *testing.T) { t.Fatalf("templates should be the same:\n4.1: %s\n4.1.0: %s\n4.1.1: %s", tmpl41, tmpl410, tmpl411) } - tmpl420, err := getCseTemplate(*v420, "rde") + tmpl420, err := getCseTemplate(*v421, "rde") if err != nil { t.Fatal(err) } if tmpl420 == "" { - t.Fatalf("the obtained template for %s is empty", v420.String()) + t.Fatalf("the obtained template for %s is empty", v421.String()) } _, err = getCseTemplate(*v40, "rde") From 45b7864a5d0d786909f8d57281350b179e17bce9 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Thu, 7 Mar 2024 09:41:07 +0100 Subject: [PATCH 114/115] Fix suggestions Signed-off-by: abarreiro --- govcd/cse.go | 12 +++++++----- govcd/cse_internal.go | 4 ++-- govcd/cse_yaml.go | 5 +++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/govcd/cse.go b/govcd/cse.go index 454f99881..68ed5ebcd 100644 --- a/govcd/cse.go +++ b/govcd/cse.go @@ -44,6 +44,11 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterSettings CseClusterSettin return "", fmt.Errorf("CseCreateKubernetesClusterAsync cannot be called on a nil Organization receiver") } + tenantContext, err := org.getTenantContext() + if err != nil { + return "", fmt.Errorf("error creating the CSE Kubernetes cluster: %s", err) + } + cseSubcomponents, err := getCseComponentsVersions(clusterSettings.CseVersion) if err != nil { return "", err @@ -54,7 +59,7 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterSettings CseClusterSettin return "", fmt.Errorf("error creating the CSE Kubernetes cluster: %s", err) } - payload, err := internalSettings.getUnmarshaledRdePayload() + payload, err := internalSettings.getUnmarshalledRdePayload() if err != nil { return "", err } @@ -64,10 +69,7 @@ func (org *Org) CseCreateKubernetesClusterAsync(clusterSettings CseClusterSettin EntityType: internalSettings.RdeType.ID, Name: internalSettings.Name, Entity: payload, - }, &TenantContext{ - OrgId: org.Org.ID, - OrgName: org.Org.Name, - }) + }, tenantContext) if err != nil { return "", fmt.Errorf("error creating the CSE Kubernetes cluster: %s", err) } diff --git a/govcd/cse_internal.go b/govcd/cse_internal.go index 0adfd83d3..d0ebb455b 100644 --- a/govcd/cse_internal.go +++ b/govcd/cse_internal.go @@ -16,9 +16,9 @@ import ( //go:embed cse var cseFiles embed.FS -// getUnmarshaledRdePayload gets the unmarshaled JSON payload to create the Runtime Defined Entity that represents +// getUnmarshalledRdePayload gets the unmarshalled JSON payload to create the Runtime Defined Entity that represents // a CSE Kubernetes cluster, by using the receiver information. This method uses all the Go Templates stored in cseFiles -func (clusterSettings *cseClusterSettingsInternal) getUnmarshaledRdePayload() (map[string]interface{}, error) { +func (clusterSettings *cseClusterSettingsInternal) getUnmarshalledRdePayload() (map[string]interface{}, error) { if clusterSettings == nil { return nil, fmt.Errorf("the receiver CSE Kubernetes cluster settings object is nil") } diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 12e2566bd..35aad87ed 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -13,9 +13,10 @@ import ( // If some of the values of the input is not provided, it doesn't change them. // If none of the values is provided, it just returns the same untouched YAML. func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) (string, error) { - if cluster == nil { - return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("receiver cluster is nil") + if cluster == nil || cluster.capvcdType == nil { + return "", fmt.Errorf("receiver cluster is nil") } + if input.ControlPlane == nil && input.WorkerPools == nil && input.NodeHealthCheck == nil && input.KubernetesTemplateOvaId == nil && input.NewWorkerPools == nil { return cluster.capvcdType.Spec.CapiYaml, nil } From 95cd3aa66a0b35e8037fd664feebfe694147d651 Mon Sep 17 00:00:00 2001 From: abarreiro Date: Fri, 8 Mar 2024 09:26:04 +0100 Subject: [PATCH 115/115] Fix typos and hide API token in logs Signed-off-by: abarreiro --- govcd/cse_util.go | 2 +- govcd/cse_yaml.go | 4 ++-- util/logging.go | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/govcd/cse_util.go b/govcd/cse_util.go index 6c7d244e3..852bcf4f0 100644 --- a/govcd/cse_util.go +++ b/govcd/cse_util.go @@ -793,7 +793,7 @@ func getTkgVersionBundleFromVAppTemplate(template *types.VAppTemplate) (tkgVersi versionsMap := map[string]interface{}{} err = json.Unmarshal(cseTkgVersionsJson, &versionsMap) if err != nil { - return result, fmt.Errorf("failed unmarshaling %s: %s", tkgVersionsMap, err) + return result, fmt.Errorf("failed unmarshalling %s: %s", tkgVersionsMap, err) } versionMap, ok := versionsMap[id] if !ok { diff --git a/govcd/cse_yaml.go b/govcd/cse_yaml.go index 35aad87ed..f5cc5af06 100644 --- a/govcd/cse_yaml.go +++ b/govcd/cse_yaml.go @@ -25,7 +25,7 @@ func (cluster *CseKubernetesCluster) updateCapiYaml(input CseClusterUpdateInput) // document it finds. yamlDocs, err := unmarshalMultipleYamlDocuments(cluster.capvcdType.Spec.CapiYaml) if err != nil { - return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("error unmarshaling YAML: %s", err) + return cluster.capvcdType.Spec.CapiYaml, fmt.Errorf("error unmarshalling YAML: %s", err) } if input.ControlPlane != nil { @@ -347,7 +347,7 @@ func marshalMultipleYamlDocuments(yamlDocuments []map[string]interface{}) (strin } // unmarshalMultipleYamlDocuments takes a multi-document YAML (multiple YAML documents are separated by "---") and -// unmarshals all of them into a slice of generic maps with the corresponding content. +// unmarshalls all of them into a slice of generic maps with the corresponding content. func unmarshalMultipleYamlDocuments(yamlDocuments string) ([]map[string]interface{}, error) { if len(strings.TrimSpace(yamlDocuments)) == 0 { return []map[string]interface{}{}, nil diff --git a/util/logging.go b/util/logging.go index f2a03bed9..819740d30 100644 --- a/util/logging.go +++ b/util/logging.go @@ -204,6 +204,10 @@ func hideSensitive(in string, onScreen bool) string { re9 := regexp.MustCompile(`("refresh_token":\s*)"[^"]*`) out = re9.ReplaceAllString(out, `${1}*******`) + // API Token inside CSE JSON payloads + re10 := regexp.MustCompile(`("apiToken":\s*)"[^"]*`) + out = re10.ReplaceAllString(out, `${1}*******`) + return out }