From 9061b27af2c9a03d8a23a7c531d7aec7ba745206 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Wed, 10 Jul 2024 09:57:33 -0400 Subject: [PATCH 01/30] vsphere cluster template --- templates/vsphere/cluster_template.yaml | 1464 +++++++++++++++++++++++ 1 file changed, 1464 insertions(+) create mode 100644 templates/vsphere/cluster_template.yaml diff --git a/templates/vsphere/cluster_template.yaml b/templates/vsphere/cluster_template.yaml new file mode 100644 index 00000000..b85bc714 --- /dev/null +++ b/templates/vsphere/cluster_template.yaml @@ -0,0 +1,1464 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta2 + kind: CK8sControlPlane + name: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + controlPlaneEndpoint: + host: 10.246.154.67 + port: 6443 + identityRef: + kind: Secret + name: ${CLUSTER_NAME} + server: ${VSPHERE_SERVER} + thumbprint: ${VSPHERE_TLS_THUMBPRINT} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + template: + spec: + cloneMode: linkedClone + datacenter: ${VSPHERE_DATACENTER} + datastore: ${VSPHERE_DATASTORE} + diskGiB: 25 + folder: ${VSPHERE_FOLDER} + memoryMiB: 8192 + network: + devices: + - dhcp4: true + networkName: ${VSPHERE_NETWORK} + numCPUs: 2 + os: Linux + powerOffMode: trySoft + resourcePool: '${VSPHERE_RESOURCE_POOL}' + server: ${VSPHERE_SERVER} + storagePolicyName: "" + template: ${VSPHERE_TEMPLATE} + thumbprint: ${VSPHERE_TLS_THUMBPRINT} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: ${CLUSTER_NAME}-worker + namespace: default +spec: + template: + spec: + cloneMode: linkedClone + datacenter: ${VSPHERE_DATACENTER} + datastore: ${VSPHERE_DATASTORE} + diskGiB: 25 + folder: ${VSPHERE_FOLDER} + memoryMiB: 8192 + network: + devices: + - dhcp4: true + networkName: ${VSPHERE_NETWORK} + numCPUs: 2 + os: Linux + powerOffMode: trySoft + resourcePool: '${VSPHERE_RESOURCE_POOL}' + server: ${VSPHERE_SERVER} + storagePolicyName: "" + template: ${VSPHERE_TEMPLATE} + thumbprint: ${VSPHERE_TLS_THUMBPRINT} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: CK8sControlPlane +metadata: + name: ${CLUSTER_NAME} + namespace: default + #ownerReferences: + # - apiVersion: cluster.x-k8s.io/v1beta1 + # controller: true + # kind: Cluster + # name: ${CLUSTER_NAME} + # uid: 6d61fb96-ea11-4456-8ff7-b9fa9f807637 +spec: + spec: + controlPlane: + cloudProvider: external + files: + - content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: "" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: svc_election + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: vip_address + value: "10.246.154.67" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.4 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + - mountPath: /etc/hosts + name: etchosts + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts + status: {} + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + permissions: "0644" + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + owner: root:root + path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh + permissions: "0700" + preRunCommands: + - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" + >/etc/hosts + - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost + localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - mkdir -p /etc/pre-kubeadm-commands + - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort); + do echo "Running script $script"; "$script"; done + - echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCt+HE0lC4j6JInF/KrGgoE/bzYpCYXFeC1xB+F4glA3xplYSJb6S+ua7lKR8jQpvOIj0Ju3mTVFea7/dT3esrEX8whWdOCkcX4ct8gYNRiS0veV9x8GqhS8TE1/Pnnx8yi+wd7J87Yd6ThiuDzraIijsNdP7ETOdMu3DEIN+HNU626TVl5HD/4mBP5Bj6MDm3f3saG07EFv+PYb9AtkHGrwU0lEiY3aatU4v2oJqWCUUmv4XxzfdWKG6ewoMhL7iQx4fsERcTUNsZhdfddQnfHmfcp2ttEbqUX4UWu4ol4H4MIJ8Rdt2Xh0o2DgIUZRf2UD9SCZuMRXjvhsVG2jHKwuXCnalvWJsKQK7KahYDLNyU+cWFP5NweRlYPF/VOxfcB+e+u4vQfll/IrfTDf5ohQzfJtk+h8DdRwg0dz4BMUOBqn3n04AxC7aYSljmeWeM5rdY+GzOzDu64Lhz3kkqH4FW3F2rHnRtPvOYid9I7zums3Hb80ODQd3BnBRmkRD/bFAOpnfEW0cWxF7WsWT0Xl2BCDOmG33N0IuTY0Z848xdU2iNZBXQuCFy/lIxY+ZeK5DRlhURJId8OoRS7HlL2+BvAi3fecOc2gYPld88QmyThtfYnGuuEHKRFY5IHEvhKJ8B5sy9QFNIM0XiWEuDL3/b3NfZrYUBMJXgtM/AGLw==" >> /root/.ssh/authorized_keys + - chmod 700 /root/.ssh + - chmod 600 /root/.ssh/authorized_keys + - sudo systemctl restart sshd + machineTemplate: + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereMachineTemplate + name: ${CLUSTER_NAME} + replicas: 1 + version: v1.30.0 +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: CK8sConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + controlPlane: + cloudProvider: external + preRunCommands: + - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" + >/etc/hosts + - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost + localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - mkdir -p /etc/pre-kubeadm-commands + - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort); + do echo "Running script $script"; "$script"; done +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: 3 + selector: + matchLabels: {} + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: CK8sConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereMachineTemplate + name: ${CLUSTER_NAME}-worker + version: v1.30.0 +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: ${CLUSTER_NAME}-crs-0 + namespace: default +spec: + clusterSelector: + matchLabels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + resources: + - kind: Secret + name: vsphere-config-secret + - kind: ConfigMap + name: csi-manifests + - kind: Secret + name: cloud-provider-vsphere-credentials + - kind: ConfigMap + name: cpi-manifests +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME} + namespace: default +stringData: + password: ${VSPHERE_PASSWORD} + username: ${VSPHERE_USERNAME} +--- +apiVersion: v1 +kind: Secret +metadata: + name: vsphere-config-secret + namespace: default +stringData: + data: | + apiVersion: v1 + kind: Secret + metadata: + name: vsphere-config-secret + namespace: vmware-system-csi + stringData: + csi-vsphere.conf: |+ + [Global] + thumbprint = "${VSPHERE_TLS_THUMBPRINT}" + + [VirtualCenter "${VSPHERE_SERVER}"] + user = "${VSPHERE_USERNAME}" + password = "${VSPHERE_PASSWORD}" + datacenters = "${VSPHERE_DATACENTER}" + + [Network] + public-network = "${VSPHERE_NETWORK}" + + type: Opaque +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + data: | + --- + apiVersion: v1 + kind: Namespace + metadata: + name: vmware-system-csi + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + name: csi.vsphere.vmware.com + spec: + attachRequired: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-controller + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-controller-role + rules: + - apiGroups: + - "" + resources: + - nodes + - pods + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - patch + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create + - update + - patch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - patch + - apiGroups: + - cns.vmware.com + resources: + - triggercsifullsyncs + verbs: + - create + - get + - update + - watch + - list + - apiGroups: + - cns.vmware.com + resources: + - cnsvspherevolumemigrations + verbs: + - create + - get + - list + - watch + - update + - delete + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeinfoes + verbs: + - create + - get + - list + - watch + - delete + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - create + - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + - apiGroups: + - cns.vmware.com + resources: + - cnsvolumeoperationrequests + verbs: + - create + - get + - list + - update + - delete + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - watch + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - get + - update + - watch + - list + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-controller-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-controller-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-controller + namespace: vmware-system-csi + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: vsphere-csi-node-cluster-role + rules: + - apiGroups: + - cns.vmware.com + resources: + - csinodetopologies + verbs: + - create + - watch + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: vsphere-csi-node-cluster-role-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vsphere-csi-node-cluster-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: vsphere-csi-node-role + namespace: vmware-system-csi + rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: vsphere-csi-node-binding + namespace: vmware-system-csi + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vsphere-csi-node-role + subjects: + - kind: ServiceAccount + name: vsphere-csi-node + namespace: vmware-system-csi + --- + apiVersion: v1 + data: + async-query-volume: "true" + block-volume-snapshot: "true" + cnsmgr-suspend-create-volume: "true" + csi-auth-check: "true" + csi-internal-generated-cluster-id: "true" + csi-migration: "true" + csi-windows-support: "true" + list-volumes: "true" + listview-tasks: "true" + max-pvscsi-targets-per-vm: "true" + multi-vcenter-csi-topology: "true" + online-volume-extend: "true" + pv-to-backingdiskobjectid-mapping: "false" + topology-preferential-datastores: "true" + trigger-csi-fullsync: "false" + kind: ConfigMap + metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: vmware-system-csi + --- + apiVersion: v1 + kind: Service + metadata: + labels: + app: vsphere-csi-controller + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + ports: + - name: ctlr + port: 2112 + protocol: TCP + targetPort: 2112 + - name: syncer + port: 2113 + protocol: TCP + targetPort: 2113 + selector: + app: vsphere-csi-controller + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: vsphere-csi-controller + namespace: vmware-system-csi + spec: + replicas: 1 + selector: + matchLabels: + app: vsphere-csi-controller + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app: vsphere-csi-controller + role: vsphere-csi + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vsphere-csi-controller + topologyKey: kubernetes.io/hostname + containers: + - args: + + - --logtostderr + - -v=6 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 + name: csi-attacher + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=4 + - --timeout=300s + - --handle-volume-inuse-error=false + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + name: csi-resizer + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: X_CSI_MODE + value: controller + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m + - name: VSPHERE_CSI_CONFIG + value: /etc/cloud/csi-vsphere.conf + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + periodSeconds: 180 + timeoutSeconds: 10 + name: vsphere-csi-controller + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + - containerPort: 2112 + name: prometheus + protocol: TCP + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - mountPath: /csi + name: socket-dir + - args: + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --leader-election + - --leader-election-lease-duration=30s + - --leader-election-renew-deadline=20s + - --leader-election-retry-period=10s + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: FULL_SYNC_INTERVAL_MINUTES + value: "30" + - name: VSPHERE_CSI_CONFIG + value: /etc/cloud/csi-vsphere.conf + - name: LOGGER_LEVEL + value: PRODUCTION + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 + imagePullPolicy: Always + name: vsphere-syncer + ports: + - containerPort: 2113 + name: prometheus + protocol: TCP + securityContext: + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - args: + - --v=4 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --kube-api-qps=100 + - --kube-api-burst=100 + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + name: csi-provisioner + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=4 + - --kube-api-qps=100 + - --kube-api-burst=100 + - --timeout=300s + - --csi-address=$(ADDRESS) + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + env: + - name: ADDRESS + value: /csi/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + name: csi-snapshotter + volumeMounts: + - mountPath: /csi + name: socket-dir + dnsPolicy: Default + priorityClassName: system-cluster-critical + serviceAccountName: vsphere-csi-controller + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - name: vsphere-config-volume + secret: + secretName: vsphere-config-secret + - emptyDir: {} + name: socket-dir + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: vsphere-csi-node + namespace: vmware-system-csi + spec: + selector: + matchLabels: + app: vsphere-csi-node + template: + metadata: + labels: + app: vsphere-csi-node + role: vsphere-csi + spec: + containers: + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" + - name: X_CSI_MODE + value: node + - name: X_CSI_SPEC_REQ_VALIDATION + value: "false" + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: LOGGER_LEVEL + value: PRODUCTION + - name: GODEBUG + value: x509sha1=1 + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + name: vsphere-csi-node + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-mount-dir + - mountPath: /dev + name: device-dir + - mountPath: /sys/block + name: blocks-dir + - mountPath: /sys/devices + name: sys-devices-dir + - args: + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: /dev + name: device-dir + - hostPath: + path: /sys/block + type: Directory + name: blocks-dir + - hostPath: + path: /sys/devices + type: Directory + name: sys-devices-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: vsphere-csi-node-windows + namespace: vmware-system-csi + spec: + selector: + matchLabels: + app: vsphere-csi-node-windows + template: + metadata: + labels: + app: vsphere-csi-node-windows + role: vsphere-csi-windows + spec: + containers: + - args: + - --v=5 + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: ADDRESS + value: unix://C:\\csi\\csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock + - --mode=kubelet-registration-probe + initialDelaySeconds: 3 + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --fss-name=internal-feature-states.csi.vsphere.vmware.com + - --fss-namespace=$(CSI_NAMESPACE) + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + - name: MAX_VOLUMES_PER_NODE + value: "59" + - name: X_CSI_MODE + value: node + - name: X_CSI_SPEC_REQ_VALIDATION + value: "false" + - name: X_CSI_SPEC_DISABLE_LEN_CHECK + value: "true" + - name: LOGGER_LEVEL + value: PRODUCTION + - name: X_CSI_LOG_LEVEL + value: DEBUG + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES + value: "1" + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + name: vsphere-csi-node + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + volumeMounts: + - mountPath: C:\csi + name: plugin-dir + - mountPath: C:\var\lib\kubelet + name: pods-mount-dir + - mountPath: \\.\pipe\csi-proxy-volume-v1 + name: csi-proxy-volume-v1 + - mountPath: \\.\pipe\csi-proxy-filesystem-v1 + name: csi-proxy-filesystem-v1 + - mountPath: \\.\pipe\csi-proxy-disk-v1 + name: csi-proxy-disk-v1 + - mountPath: \\.\pipe\csi-proxy-system-v1alpha1 + name: csi-proxy-system-v1alpha1 + - args: + - --v=4 + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: windows + priorityClassName: system-node-critical + serviceAccountName: vsphere-csi-node + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: C:\var\lib\kubelet\plugins_registry\ + type: Directory + name: registration-dir + - hostPath: + path: C:\var\lib\kubelet\plugins\csi.vsphere.vmware.com\ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: \var\lib\kubelet + type: Directory + name: pods-mount-dir + - hostPath: + path: \\.\pipe\csi-proxy-disk-v1 + type: "" + name: csi-proxy-disk-v1 + - hostPath: + path: \\.\pipe\csi-proxy-volume-v1 + type: "" + name: csi-proxy-volume-v1 + - hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + type: "" + name: csi-proxy-filesystem-v1 + - hostPath: + path: \\.\pipe\csi-proxy-system-v1alpha1 + type: "" + name: csi-proxy-system-v1alpha1 + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate +kind: ConfigMap +metadata: + name: csi-manifests + namespace: default +--- +apiVersion: v1 +kind: Secret +metadata: + name: cloud-provider-vsphere-credentials + namespace: default +stringData: + data: | + apiVersion: v1 + kind: Secret + metadata: + labels: + component: cloud-controller-manager + vsphere-cpi-infra: secret + name: cloud-provider-vsphere-credentials + namespace: kube-system + stringData: + ${VSPHERE_SERVER}.password: ${VSPHERE_PASSWORD} + ${VSPHERE_SERVER}.username: ${VSPHERE_USERNAME} + type: Opaque +type: addons.cluster.x-k8s.io/resource-set +--- +apiVersion: v1 +data: + data: | + --- + # Source: vsphere-cpi/templates/service-account.yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + labels: + app: vsphere-cpi + vsphere-cpi-infra: service-account + component: cloud-controller-manager + namespace: kube-system + --- + # Source: vsphere-cpi/templates/role.yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: cloud-controller-manager + labels: + app: vsphere-cpi + vsphere-cpi-infra: role + component: cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + # Source: vsphere-cpi/templates/daemonset.yaml + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: vsphere-cpi + labels: + app: vsphere-cpi + vsphere-cpi-infra: daemonset + component: cloud-controller-manager + tier: control-plane + namespace: kube-system + annotations: + spec: + selector: + matchLabels: + app: vsphere-cpi + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: vsphere-cpi + component: cloud-controller-manager + tier: control-plane + release: release-name + vsphere-cpi-infra: daemonset + spec: + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + operator: Exists + - key: node.kubernetes.io/not-ready + effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + effect: NoExecute + operator: Exists + securityContext: + fsGroup: 1001 + runAsUser: 1001 + serviceAccountName: cloud-controller-manager + hostNetwork: true + dnsPolicy: ClusterFirst + priorityClassName: system-node-critical + containers: + - name: vsphere-cpi + image: gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.27.13 + imagePullPolicy: IfNotPresent + args: + - --cloud-provider=vsphere + - --v=2 + - --cloud-config=/etc/cloud/vsphere.conf + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + volumes: + - name: vsphere-config-volume + configMap: + name: cloud-config + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + app: vsphere-cpi + component: cloud-controller-manager + vsphere-cpi-infra: role-binding + name: servicecatalog.k8s.io:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app: vsphere-cpi + component: cloud-controller-manager + vsphere-cpi-infra: cluster-role-binding + name: cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloud-controller-manager + subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - kind: User + name: cloud-controller-manager + --- + apiVersion: v1 + data: + vsphere.conf: | + global: + port: 443 + secretName: cloud-provider-vsphere-credentials + secretNamespace: kube-system + thumbprint: '${VSPHERE_TLS_THUMBPRINT}' + vcenter: + ${VSPHERE_SERVER}: + datacenters: + - '${VSPHERE_DATACENTER}' + server: '${VSPHERE_SERVER}' + kind: ConfigMap + metadata: + name: cloud-config + namespace: kube-system +kind: ConfigMap +metadata: + name: cpi-manifests + namespace: default From 39e6e5e9fd20e1ea4c29f0c9d3da38f2fc50165b Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 12 Jul 2024 11:57:02 -0400 Subject: [PATCH 02/30] replace hard-coded values by config --- templates/vsphere/cluster_template.yaml | 38 +++++++++++++++---------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/templates/vsphere/cluster_template.yaml b/templates/vsphere/cluster_template.yaml index b85bc714..8e013b77 100644 --- a/templates/vsphere/cluster_template.yaml +++ b/templates/vsphere/cluster_template.yaml @@ -26,7 +26,7 @@ metadata: namespace: default spec: controlPlaneEndpoint: - host: 10.246.154.67 + host: ${CONTROL_PLANE_ENDPOINT_IP} port: 6443 identityRef: kind: Secret @@ -93,17 +93,22 @@ kind: CK8sControlPlane metadata: name: ${CLUSTER_NAME} namespace: default - #ownerReferences: - # - apiVersion: cluster.x-k8s.io/v1beta1 - # controller: true - # kind: Cluster - # name: ${CLUSTER_NAME} - # uid: 6d61fb96-ea11-4456-8ff7-b9fa9f807637 spec: spec: controlPlane: cloudProvider: external files: + - content: | + [Service] + Environment="HTTPS_PROXY=http://squid.internal:3128" + Environment="HTTP_PROXY=http://squid.internal:3128" + Environment="https_proxy=http://squid.internal:3128" + Environment="http_proxy=http://squid.internal:3128" + Environment="NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" + Environment="no_proxy=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" + owner: root:root + path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.conf + permissions: "0644" - content: | apiVersion: v1 kind: Pod @@ -121,7 +126,7 @@ spec: - name: port value: "6443" - name: vip_interface - value: "" + value: "${VIP_NETWORK_INTERFACE}" - name: vip_cidr value: "32" - name: cp_enable @@ -147,7 +152,7 @@ spec: - name: vip_retryperiod value: "2" - name: vip_address - value: "10.246.154.67" + value: "${CONTROL_PLANE_ENDPOINT_IP}" - name: prometheus_server value: :2112 image: ghcr.io/kube-vip/kube-vip:v0.6.4 @@ -204,8 +209,8 @@ spec: # xref: https://github.com/kube-vip/kube-vip/issues/684 # Nothing to do for kubernetes < v1.29 - sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ - /etc/kubernetes/manifests/kube-vip.yaml + #sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + #/etc/kubernetes/manifests/kube-vip.yaml owner: root:root path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh permissions: "0700" @@ -222,13 +227,16 @@ spec: - chmod 700 /root/.ssh - chmod 600 /root/.ssh/authorized_keys - sudo systemctl restart sshd + - mkdir -p /etc/systemd/system/snap.k8s.containerd.service.d + - sudo snap install k8s --classic --edge + machineTemplate: infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate name: ${CLUSTER_NAME} - replicas: 1 - version: v1.30.0 + replicas: 3 + version: ${K8S_VERSION} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate @@ -277,7 +285,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate name: ${CLUSTER_NAME}-worker - version: v1.30.0 + version: ${K8S_VERSION} --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet @@ -1375,7 +1383,7 @@ data: priorityClassName: system-node-critical containers: - name: vsphere-cpi - image: gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.27.13 + image: gcr.io/cloud-provider-vsphere/cpi/release/manager:${CPI_IMAGE_K8S_VERSION} imagePullPolicy: IfNotPresent args: - --cloud-provider=vsphere From 55a6dcd375600bced9896d6daa7b0911d24663ef Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 12 Jul 2024 15:29:38 -0400 Subject: [PATCH 03/30] add uninitialized toleration nodes are tainted with node.cloudprovider.kubernetes.io/uninitialized because we set cloudProvider: external. this prevents some crucial pods from being scheduled (for example we need the kube-vip pod to start on the first node so it can announce the control plane IP) --- templates/vsphere/cluster_template.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/templates/vsphere/cluster_template.yaml b/templates/vsphere/cluster_template.yaml index 8e013b77..a5166b63 100644 --- a/templates/vsphere/cluster_template.yaml +++ b/templates/vsphere/cluster_template.yaml @@ -117,6 +117,10 @@ spec: name: kube-vip namespace: kube-system spec: + tolerations: + - key: "node.cloudprovider.kubernetes.io/uninitialized" + operator: "Exists" + effect: "NoSchedule" containers: - args: - manager @@ -896,6 +900,10 @@ data: - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists + #https://github.com/kubernetes-sigs/vsphere-csi-driver/issues/2284 + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule volumes: - name: vsphere-config-volume secret: From fc98d1b11b0a73339260a5eff2f77f7935cb4e38 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 12 Jul 2024 15:32:55 -0400 Subject: [PATCH 04/30] cleanup leftover kubeadm scripts --- templates/vsphere/cluster_template.yaml | 36 +------------------------ 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/templates/vsphere/cluster_template.yaml b/templates/vsphere/cluster_template.yaml index a5166b63..7e845eb7 100644 --- a/templates/vsphere/cluster_template.yaml +++ b/templates/vsphere/cluster_template.yaml @@ -190,44 +190,13 @@ spec: owner: root:root path: /etc/kube-vip.hosts permissions: "0644" - - content: | - #!/bin/bash - - # Copyright 2020 The Kubernetes Authors. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - set -e - - # Configure the workaround required for kubeadm init with kube-vip: - # xref: https://github.com/kube-vip/kube-vip/issues/684 - - # Nothing to do for kubernetes < v1.29 - #sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ - #/etc/kubernetes/manifests/kube-vip.yaml - owner: root:root - path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh - permissions: "0700" preRunCommands: - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts - - mkdir -p /etc/pre-kubeadm-commands - - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort); - do echo "Running script $script"; "$script"; done - - echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCt+HE0lC4j6JInF/KrGgoE/bzYpCYXFeC1xB+F4glA3xplYSJb6S+ua7lKR8jQpvOIj0Ju3mTVFea7/dT3esrEX8whWdOCkcX4ct8gYNRiS0veV9x8GqhS8TE1/Pnnx8yi+wd7J87Yd6ThiuDzraIijsNdP7ETOdMu3DEIN+HNU626TVl5HD/4mBP5Bj6MDm3f3saG07EFv+PYb9AtkHGrwU0lEiY3aatU4v2oJqWCUUmv4XxzfdWKG6ewoMhL7iQx4fsERcTUNsZhdfddQnfHmfcp2ttEbqUX4UWu4ol4H4MIJ8Rdt2Xh0o2DgIUZRf2UD9SCZuMRXjvhsVG2jHKwuXCnalvWJsKQK7KahYDLNyU+cWFP5NweRlYPF/VOxfcB+e+u4vQfll/IrfTDf5ohQzfJtk+h8DdRwg0dz4BMUOBqn3n04AxC7aYSljmeWeM5rdY+GzOzDu64Lhz3kkqH4FW3F2rHnRtPvOYid9I7zums3Hb80ODQd3BnBRmkRD/bFAOpnfEW0cWxF7WsWT0Xl2BCDOmG33N0IuTY0Z848xdU2iNZBXQuCFy/lIxY+ZeK5DRlhURJId8OoRS7HlL2+BvAi3fecOc2gYPld88QmyThtfYnGuuEHKRFY5IHEvhKJ8B5sy9QFNIM0XiWEuDL3/b3NfZrYUBMJXgtM/AGLw==" >> /root/.ssh/authorized_keys + - echo "${VSPHERE_SSH_AUTHORIZED_KEY}" >> /root/.ssh/authorized_keys - chmod 700 /root/.ssh - chmod 600 /root/.ssh/authorized_keys - sudo systemctl restart sshd @@ -258,9 +227,6 @@ spec: >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts - - mkdir -p /etc/pre-kubeadm-commands - - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort); - do echo "Running script $script"; "$script"; done --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment From d48199bb016495857ff02621bec77f284fa9017a Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 12 Jul 2024 15:36:40 -0400 Subject: [PATCH 05/30] workaround to "thumbprint does not match" set insecure-flag="true" to get around a possible thumbprint verification bug in vsphere-csi driver. Also update csi driver versions to v3.3.0 which support k8s v1.28-v1.30. --- templates/vsphere/cluster_template.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/vsphere/cluster_template.yaml b/templates/vsphere/cluster_template.yaml index 7e845eb7..cda060c3 100644 --- a/templates/vsphere/cluster_template.yaml +++ b/templates/vsphere/cluster_template.yaml @@ -303,6 +303,9 @@ stringData: csi-vsphere.conf: |+ [Global] thumbprint = "${VSPHERE_TLS_THUMBPRINT}" + insecure-flag = "true" # todo - set to false causes in csi-controller pod "error thumbprint does not match". + # Investigate: https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/3.0/vmware-vsphere-csp-getting-started/GUID-BFF39F1D-F70A-4360-ABC9-85BDAFBE8864.html + # and https://docs.vmware.com/en/VMware-vSphere-Container-Storage-Plug-in/3.0/vmware-vsphere-csp-getting-started/GUID-BFF39F1D-F70A-4360-ABC9-85BDAFBE8864.html#GUID-D1F411D8-FE2E-466D-B85B-EEC847385E61 [VirtualCenter "${VSPHERE_SERVER}"] user = "${VSPHERE_USERNAME}" @@ -744,7 +747,7 @@ data: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.3.0 imagePullPolicy: Always livenessProbe: failureThreshold: 3 @@ -944,7 +947,7 @@ data: fieldPath: metadata.namespace - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES value: "1" - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 + image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.3.0 imagePullPolicy: Always livenessProbe: failureThreshold: 3 From ac9348c0b9f7aef006ed96ec2ed3f668fa09d22f Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 12 Jul 2024 15:38:48 -0400 Subject: [PATCH 06/30] hack: install k8s manually, enable kube-vip --- templates/vsphere/cluster_template.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/vsphere/cluster_template.yaml b/templates/vsphere/cluster_template.yaml index cda060c3..f57c9988 100644 --- a/templates/vsphere/cluster_template.yaml +++ b/templates/vsphere/cluster_template.yaml @@ -199,9 +199,10 @@ spec: - echo "${VSPHERE_SSH_AUTHORIZED_KEY}" >> /root/.ssh/authorized_keys - chmod 700 /root/.ssh - chmod 600 /root/.ssh/authorized_keys - - sudo systemctl restart sshd - mkdir -p /etc/systemd/system/snap.k8s.containerd.service.d - - sudo snap install k8s --classic --edge + - snap install k8s --classic --edge # todo why not installed automatically? + postRunCommands: + - k8s kubectl apply -f /etc/kubernetes/manifests/kube-vip.yaml # todo how to apply this properly machineTemplate: infrastructureTemplate: From 396fe6b02d9ac7e9d663f1b844a00ae4eea3b1fd Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 12 Jul 2024 15:45:47 -0400 Subject: [PATCH 07/30] add template-variables file --- templates/vsphere/template-variables.rc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 templates/vsphere/template-variables.rc diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc new file mode 100644 index 00000000..1d57520c --- /dev/null +++ b/templates/vsphere/template-variables.rc @@ -0,0 +1,23 @@ +# Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md + +## -- Controller settings -- ## +VSPHERE_USERNAME: "vi-admin@vsphere.local" # The username used to access the remote vSphere endpoint +VSPHERE_PASSWORD: "admin!23" # The password used to access the remote vSphere endpoint + +## -- Required workload cluster default settings -- ## +VSPHERE_SERVER: "10.0.0.1" # The vCenter server IP or FQDN +VSPHERE_DATACENTER: "SDDC-Datacenter" # The vSphere datacenter to deploy the management cluster on +VSPHERE_DATASTORE: "DefaultDatastore" # The vSphere datastore to deploy the management cluster on +VSPHERE_NETWORK: "VM Network" # The VM network to deploy the management cluster on +VSPHERE_RESOURCE_POOL: "*/Resources" # The vSphere resource pool for your VMs +VSPHERE_FOLDER: "vm" # The VM folder for your VMs. Set to "" to use the root vSphere folder +VSPHERE_TEMPLATE: "ubuntu-1804-kube-v1.17.3" # The VM template to use for your management cluster. +CONTROL_PLANE_ENDPOINT_IP: "192.168.9.230" # the IP that kube-vip is going to use as a control plane endpoint +VIP_NETWORK_INTERFACE: "ens192" # The interface that kube-vip should apply the IP to. Omit to tell kube-vip to autodetect the interface. +VSPHERE_TLS_THUMBPRINT: "..." # sha256 thumbprint of the vcenter certificate: openssl x509 -sha256 -fingerprint -in ca.crt -noout +EXP_CLUSTER_RESOURCE_SET: "true" # This enables the ClusterResourceSet feature that we are using to deploy CSI +VSPHERE_SSH_AUTHORIZED_KEY: "ssh-rsa AAAAB3N..." # The public ssh authorized key on all machines in this cluster. + # Set to "" if you don't want to enable SSH, or are using another solution. +VSPHERE_STORAGE_POLICY: "" # This is the vSphere storage policy. Set it to "" if you don't want to use a storage policy. +"CPI_IMAGE_K8S_VERSION": "v1.25.0" # The version of the vSphere CPI image to be used by the CPI workloads + # Keep this close to the minimum Kubernetes version of the cluster being created. From 3159998a7ead2f3fac7833768df4b33bb8729cf8 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Fri, 12 Jul 2024 15:48:27 -0400 Subject: [PATCH 08/30] rename file, add custom variable being used --- .../vsphere/{cluster_template.yaml => cluster-template.yaml} | 0 templates/vsphere/template-variables.rc | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) rename templates/vsphere/{cluster_template.yaml => cluster-template.yaml} (100%) diff --git a/templates/vsphere/cluster_template.yaml b/templates/vsphere/cluster-template.yaml similarity index 100% rename from templates/vsphere/cluster_template.yaml rename to templates/vsphere/cluster-template.yaml diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index 1d57520c..d7b518db 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -19,5 +19,6 @@ EXP_CLUSTER_RESOURCE_SET: "true" # This enables the VSPHERE_SSH_AUTHORIZED_KEY: "ssh-rsa AAAAB3N..." # The public ssh authorized key on all machines in this cluster. # Set to "" if you don't want to enable SSH, or are using another solution. VSPHERE_STORAGE_POLICY: "" # This is the vSphere storage policy. Set it to "" if you don't want to use a storage policy. -"CPI_IMAGE_K8S_VERSION": "v1.25.0" # The version of the vSphere CPI image to be used by the CPI workloads +"CPI_IMAGE_K8S_VERSION": "v1.28.0" # The version of the vSphere CPI image to be used by the CPI workloads # Keep this close to the minimum Kubernetes version of the cluster being created. +K8S_VERSION: "v1.28.0" From f43d86d94ad8fee9db9d5681210b44b13553e7fc Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 15 Jul 2024 12:09:35 -0400 Subject: [PATCH 09/30] configure kube-vip daemonset create manifest at /capi/manifests so it's applied after bootstrap --- templates/vsphere/cluster-template.yaml | 159 +++++++++++++----------- 1 file changed, 87 insertions(+), 72 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index f57c9988..558f2d4d 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -110,81 +110,99 @@ spec: path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.conf permissions: "0644" - content: | - apiVersion: v1 - kind: Pod + apiVersion: apps/v1 + kind: DaemonSet metadata: creationTimestamp: null name: kube-vip namespace: kube-system spec: - tolerations: - - key: "node.cloudprovider.kubernetes.io/uninitialized" - operator: "Exists" - effect: "NoSchedule" - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: "${VIP_NETWORK_INTERFACE}" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: svc_leasename - value: plndr-svcs-lock - - name: svc_election - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leasename - value: plndr-cp-lock - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: vip_address - value: "${CONTROL_PLANE_ENDPOINT_IP}" - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.6.4 - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - - mountPath: /etc/hosts - name: etchosts - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - name: kubeconfig - - hostPath: - path: /etc/kube-vip.hosts - type: File - name: etchosts + selector: + matchLabels: + name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + name: kube-vip-ds + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: "${VIP_NETWORK_INTERFACE}" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: svc_election + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: vip_address + value: "${CONTROL_PLANE_ENDPOINT_IP}" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.4 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + - mountPath: /etc/hosts + name: etchosts + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} status: {} owner: root:root - path: /etc/kubernetes/manifests/kube-vip.yaml + path: /capi/manifests/kube-vip.yaml permissions: "0644" - content: 127.0.0.1 localhost kubernetes owner: root:root @@ -200,10 +218,7 @@ spec: - chmod 700 /root/.ssh - chmod 600 /root/.ssh/authorized_keys - mkdir -p /etc/systemd/system/snap.k8s.containerd.service.d - - snap install k8s --classic --edge # todo why not installed automatically? - postRunCommands: - - k8s kubectl apply -f /etc/kubernetes/manifests/kube-vip.yaml # todo how to apply this properly - + - sudo snap install k8s --classic --edge machineTemplate: infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 From b4e0279f86bd71bc0f05443c067c86a2229eee74 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 15 Jul 2024 12:11:49 -0400 Subject: [PATCH 10/30] remove node-role/master toleration/affinity --- templates/vsphere/cluster-template.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 558f2d4d..f6826894 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -879,9 +879,6 @@ data: priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists @@ -1355,8 +1352,6 @@ data: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - - key: node-role.kubernetes.io/master - effect: NoSchedule operator: Exists - key: node-role.kubernetes.io/control-plane effect: NoSchedule @@ -1397,9 +1392,6 @@ data: - matchExpressions: - key: node-role.kubernetes.io/control-plane operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From 998ebfcd6af05a3780ba19f9b255187186b0e7e0 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 15 Jul 2024 12:14:51 -0400 Subject: [PATCH 11/30] yaml -> bash --- templates/vsphere/template-variables.rc | 40 ++++++++++++------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index d7b518db..342c70d0 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -1,24 +1,22 @@ # Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md -## -- Controller settings -- ## -VSPHERE_USERNAME: "vi-admin@vsphere.local" # The username used to access the remote vSphere endpoint -VSPHERE_PASSWORD: "admin!23" # The password used to access the remote vSphere endpoint +export VSPHERE_USERNAME="vi-admin@vsphere.local" # The username used to access the remote vSphere endpoint +export VSPHERE_PASSWORD="admin!23" # The password used to access the remote vSphere endpoint -## -- Required workload cluster default settings -- ## -VSPHERE_SERVER: "10.0.0.1" # The vCenter server IP or FQDN -VSPHERE_DATACENTER: "SDDC-Datacenter" # The vSphere datacenter to deploy the management cluster on -VSPHERE_DATASTORE: "DefaultDatastore" # The vSphere datastore to deploy the management cluster on -VSPHERE_NETWORK: "VM Network" # The VM network to deploy the management cluster on -VSPHERE_RESOURCE_POOL: "*/Resources" # The vSphere resource pool for your VMs -VSPHERE_FOLDER: "vm" # The VM folder for your VMs. Set to "" to use the root vSphere folder -VSPHERE_TEMPLATE: "ubuntu-1804-kube-v1.17.3" # The VM template to use for your management cluster. -CONTROL_PLANE_ENDPOINT_IP: "192.168.9.230" # the IP that kube-vip is going to use as a control plane endpoint -VIP_NETWORK_INTERFACE: "ens192" # The interface that kube-vip should apply the IP to. Omit to tell kube-vip to autodetect the interface. -VSPHERE_TLS_THUMBPRINT: "..." # sha256 thumbprint of the vcenter certificate: openssl x509 -sha256 -fingerprint -in ca.crt -noout -EXP_CLUSTER_RESOURCE_SET: "true" # This enables the ClusterResourceSet feature that we are using to deploy CSI -VSPHERE_SSH_AUTHORIZED_KEY: "ssh-rsa AAAAB3N..." # The public ssh authorized key on all machines in this cluster. - # Set to "" if you don't want to enable SSH, or are using another solution. -VSPHERE_STORAGE_POLICY: "" # This is the vSphere storage policy. Set it to "" if you don't want to use a storage policy. -"CPI_IMAGE_K8S_VERSION": "v1.28.0" # The version of the vSphere CPI image to be used by the CPI workloads - # Keep this close to the minimum Kubernetes version of the cluster being created. -K8S_VERSION: "v1.28.0" +export VSPHERE_SERVER="10.0.0.1" # The vCenter server IP or FQDN +export VSPHERE_DATACENTER="SDDC-Datacenter" # The vSphere datacenter to deploy the management cluster on +export VSPHERE_DATASTORE="DefaultDatastore" # The vSphere datastore to deploy the management cluster on +export VSPHERE_NETWORK="VM Network" # The VM network to deploy the management cluster on +export VSPHERE_RESOURCE_POOL="*/Resources" # The vSphere resource pool for your VMs +export VSPHERE_FOLDER="vm" # The VM folder for your VMs. Set to "" to use the root vSphere folder +export VSPHERE_TEMPLATE="ubuntu-1804-kube-v1.17.3" # The VM template to use for your management cluster. +export CONTROL_PLANE_ENDPOINT_IP="192.168.9.230" # the IP that kube-vip is going to use as a control plane endpoint +export VIP_NETWORK_INTERFACE="ens192" # The interface that kube-vip should apply the IP to. Omit to tell kube-vip to autodetect the interface. +export VSPHERE_TLS_THUMBPRINT="..." # sha256 thumbprint of the vcenter certificate: openssl x509 -sha256 -fingerprint -in ca.crt -noout +export EXP_CLUSTER_RESOURCE_SET="true" # This enables the ClusterResourceSet feature that we are using to deploy CSI +export VSPHERE_SSH_AUTHORIZED_KEY="ssh-rsa AAAAB3N..." # The public ssh authorized key on all machines in this cluster. + # Set to "" if you don't want to enable SSH, or are using another solution. +export VSPHERE_STORAGE_POLICY="" # This is the vSphere storage policy. Set it to "" if you don't want to use a storage policy. +export "CPI_IMAGE_K8S_VERSION"="v1.28.0" # The version of the vSphere CPI image to be used by the CPI workloads + +export K8S_VERSION="v1.28.0" From f447c107278f388d8d101902f6d2745db9aa38e4 Mon Sep 17 00:00:00 2001 From: eaudetcobello <155978570+eaudetcobello@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:20:15 -0400 Subject: [PATCH 12/30] hack: change sha256 to sha1 in template-variables.rc The provider acts weird when it comes to checking the thumbprint. The provider suggests to use sha256 for the thumbprint, but kubectl explain VCluster says that thumbprint expects a sha1 hash. So it's clear there's some conflicting information in the documentation. Some experimenting needs to be done in regards to removing the insecure-flag and using sha256 or sha1 for the thumbprint. The error can be seen in vsphere-cpi pods, the logs complain about "thumbprint does not match". Right now there is no error because of a combination of insecure-flag: true and using sha1. Not sure which one of these solves the issue, but it's solved for now. --- templates/vsphere/template-variables.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index 342c70d0..06e10fc7 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -12,7 +12,7 @@ export VSPHERE_FOLDER="vm" # The VM fol export VSPHERE_TEMPLATE="ubuntu-1804-kube-v1.17.3" # The VM template to use for your management cluster. export CONTROL_PLANE_ENDPOINT_IP="192.168.9.230" # the IP that kube-vip is going to use as a control plane endpoint export VIP_NETWORK_INTERFACE="ens192" # The interface that kube-vip should apply the IP to. Omit to tell kube-vip to autodetect the interface. -export VSPHERE_TLS_THUMBPRINT="..." # sha256 thumbprint of the vcenter certificate: openssl x509 -sha256 -fingerprint -in ca.crt -noout +export VSPHERE_TLS_THUMBPRINT="..." # sha1 thumbprint of the vcenter certificate: openssl x509 -sha1 -fingerprint -in ca.crt -noout export EXP_CLUSTER_RESOURCE_SET="true" # This enables the ClusterResourceSet feature that we are using to deploy CSI export VSPHERE_SSH_AUTHORIZED_KEY="ssh-rsa AAAAB3N..." # The public ssh authorized key on all machines in this cluster. # Set to "" if you don't want to enable SSH, or are using another solution. From 6f9be9cd0ef934d740314659ec261f58d1c1d0a5 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 15 Jul 2024 14:09:00 -0400 Subject: [PATCH 13/30] Revert "remove node-role/master toleration/affinity" This reverts commit b4e0279f86bd71bc0f05443c067c86a2229eee74. --- templates/vsphere/cluster-template.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index f6826894..558f2d4d 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -879,6 +879,9 @@ data: priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists @@ -1352,6 +1355,8 @@ data: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule operator: Exists - key: node-role.kubernetes.io/control-plane effect: NoSchedule @@ -1392,6 +1397,9 @@ data: - matchExpressions: - key: node-role.kubernetes.io/control-plane operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From b59d593a9df276a51bd0b23435d691ecc327110b Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 15 Jul 2024 14:10:02 -0400 Subject: [PATCH 14/30] Reapply "remove node-role/master toleration/affinity" This reverts commit b3ce6ea281cc13a65dca4431a6e5a97a2355278b. --- templates/vsphere/cluster-template.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 558f2d4d..f6826894 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -879,9 +879,6 @@ data: priorityClassName: system-cluster-critical serviceAccountName: vsphere-csi-controller tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists @@ -1355,8 +1352,6 @@ data: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - - key: node-role.kubernetes.io/master - effect: NoSchedule operator: Exists - key: node-role.kubernetes.io/control-plane effect: NoSchedule @@ -1397,9 +1392,6 @@ data: - matchExpressions: - key: node-role.kubernetes.io/control-plane operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From efbec0ca012177cace010d7f394fafbeb225ca89 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Thu, 18 Jul 2024 08:45:24 -0400 Subject: [PATCH 15/30] configure proxy on workers --- templates/vsphere/cluster-template.yaml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index f6826894..61004e6b 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -243,6 +243,22 @@ spec: >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - echo "${VSPHERE_SSH_AUTHORIZED_KEY}" >> /root/.ssh/authorized_keys + - chmod 700 /root/.ssh + - chmod 600 /root/.ssh/authorized_keys + - snap install k8s --classic --edge + files: + - content: | + [Service] + Environment="HTTPS_PROXY=http://squid.internal:3128" + Environment="HTTP_PROXY=http://squid.internal:3128" + Environment="https_proxy=http://squid.internal:3128" + Environment="http_proxy=http://squid.internal:3128" + Environment="NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" + Environment="no_proxy=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" + owner: root:root + path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.conf + permissions: "0644" --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -330,7 +346,6 @@ stringData: [Network] public-network = "${VSPHERE_NETWORK}" - type: Opaque type: addons.cluster.x-k8s.io/resource-set --- @@ -823,7 +838,7 @@ data: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.1.0 + image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.3.0 imagePullPolicy: Always name: vsphere-syncer ports: @@ -884,8 +899,8 @@ data: operator: Exists #https://github.com/kubernetes-sigs/vsphere-csi-driver/issues/2284 - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" effect: NoSchedule + value: "true" volumes: - name: vsphere-config-volume secret: @@ -1352,7 +1367,6 @@ data: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" effect: NoSchedule - operator: Exists - key: node-role.kubernetes.io/control-plane effect: NoSchedule operator: Exists @@ -1442,6 +1456,7 @@ data: secretName: cloud-provider-vsphere-credentials secretNamespace: kube-system thumbprint: '${VSPHERE_TLS_THUMBPRINT}' + insecure-flag: 'true' vcenter: ${VSPHERE_SERVER}: datacenters: From d7903af04bbe03147b99539d57de3311f81b1fc5 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Thu, 18 Jul 2024 08:51:44 -0400 Subject: [PATCH 16/30] change cidrs to our standard ones --- templates/vsphere/cluster-template.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 61004e6b..8676d7ad 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -9,7 +9,10 @@ spec: clusterNetwork: pods: cidrBlocks: - - 192.168.0.0/16 + - 10.1.0.0/16 + services: + cidrBlocks: + - 10.152.183.0/24 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane From fbf7497ddafb171c33b811a60703af85a1b06298 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 22 Jul 2024 19:47:59 -0400 Subject: [PATCH 17/30] apply kube-vip rbac manifests use kube-vip rbac manifests instead of mounting the admin.conf file --- templates/vsphere/cluster-template.yaml | 48 ++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 8676d7ad..07ea5f7b 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -1,3 +1,45 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: @@ -129,6 +171,7 @@ spec: labels: name: kube-vip-ds spec: + serviceAccountName: kube-vip affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -184,15 +227,10 @@ spec: - NET_ADMIN - NET_RAW volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - mountPath: /etc/hosts name: etchosts hostNetwork: true volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - name: kubeconfig - hostPath: path: /etc/kube-vip.hosts type: File From 142a81ba77ec0a0d830b547f4b39e1359d2a044e Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 22 Jul 2024 19:49:22 -0400 Subject: [PATCH 18/30] move proxy configuration to a secret the secret can be enabled/disabled by changing VSPHERE_PROXY_DISABLE in clusterctl.yaml --- templates/vsphere/cluster-template.yaml | 38 ++++++++++++++----------- templates/vsphere/template-variables.rc | 2 +- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 07ea5f7b..b6c2ddd6 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -1,4 +1,18 @@ apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME}-vsphere-proxy-config +stringData: + proxy-config: | + ${VSPHERE_PROXY_DISABLE}[Service] + ${VSPHERE_PROXY_DISABLE}Environment="HTTPS_PROXY=http://squid.internal:3128" + ${VSPHERE_PROXY_DISABLE}Environment="HTTP_PROXY=http://squid.internal:3128" + ${VSPHERE_PROXY_DISABLE}Environment="https_proxy=http://squid.internal:3128" + ${VSPHERE_PROXY_DISABLE}Environment="http_proxy=http://squid.internal:3128" + ${VSPHERE_PROXY_DISABLE}Environment="NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" + ${VSPHERE_PROXY_DISABLE}Environment="no_proxy=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" +--- +apiVersion: v1 kind: ServiceAccount metadata: name: kube-vip @@ -143,14 +157,10 @@ spec: controlPlane: cloudProvider: external files: - - content: | - [Service] - Environment="HTTPS_PROXY=http://squid.internal:3128" - Environment="HTTP_PROXY=http://squid.internal:3128" - Environment="https_proxy=http://squid.internal:3128" - Environment="http_proxy=http://squid.internal:3128" - Environment="NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" - Environment="no_proxy=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" + - contentFrom: + secret: + key: proxy-config + name: ${CLUSTER_NAME}-vsphere-proxy-config owner: root:root path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.conf permissions: "0644" @@ -289,14 +299,10 @@ spec: - chmod 600 /root/.ssh/authorized_keys - snap install k8s --classic --edge files: - - content: | - [Service] - Environment="HTTPS_PROXY=http://squid.internal:3128" - Environment="HTTP_PROXY=http://squid.internal:3128" - Environment="https_proxy=http://squid.internal:3128" - Environment="http_proxy=http://squid.internal:3128" - Environment="NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" - Environment="no_proxy=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" + - contentFrom: + secret: + key: proxy-config + name: ${CLUSTER_NAME}-vsphere-proxy-config owner: root:root path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.conf permissions: "0644" diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index 06e10fc7..f80cc4fa 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -19,4 +19,4 @@ export VSPHERE_SSH_AUTHORIZED_KEY="ssh-rsa AAAAB3N..." # The public export VSPHERE_STORAGE_POLICY="" # This is the vSphere storage policy. Set it to "" if you don't want to use a storage policy. export "CPI_IMAGE_K8S_VERSION"="v1.28.0" # The version of the vSphere CPI image to be used by the CPI workloads -export K8S_VERSION="v1.28.0" +export VSPHERE_PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration From df145c05e27459134e0acfb27fba9a5ce60ea5e7 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 22 Jul 2024 19:50:45 -0400 Subject: [PATCH 19/30] mis changes following review --- templates/vsphere/cluster-template.yaml | 7 +++---- templates/vsphere/template-variables.rc | 10 ++++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index b6c2ddd6..e8fe6320 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -268,15 +268,14 @@ spec: - echo "${VSPHERE_SSH_AUTHORIZED_KEY}" >> /root/.ssh/authorized_keys - chmod 700 /root/.ssh - chmod 600 /root/.ssh/authorized_keys - - mkdir -p /etc/systemd/system/snap.k8s.containerd.service.d - - sudo snap install k8s --classic --edge + - snap install k8s --classic --edge machineTemplate: infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate name: ${CLUSTER_NAME} replicas: 3 - version: ${K8S_VERSION} + version: ${CK8S_VERSION} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate @@ -334,7 +333,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate name: ${CLUSTER_NAME}-worker - version: ${K8S_VERSION} + version: ${CK8S_VERSION} --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index f80cc4fa..0c54368e 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -1,7 +1,7 @@ # Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md -export VSPHERE_USERNAME="vi-admin@vsphere.local" # The username used to access the remote vSphere endpoint -export VSPHERE_PASSWORD="admin!23" # The password used to access the remote vSphere endpoint +export VSPHERE_USERNAME="" # The username used to access the remote vSphere endpoint +export VSPHERE_PASSWORD="" # The password used to access the remote vSphere endpoint export VSPHERE_SERVER="10.0.0.1" # The vCenter server IP or FQDN export VSPHERE_DATACENTER="SDDC-Datacenter" # The vSphere datacenter to deploy the management cluster on @@ -15,8 +15,10 @@ export VIP_NETWORK_INTERFACE="ens192" # The interf export VSPHERE_TLS_THUMBPRINT="..." # sha1 thumbprint of the vcenter certificate: openssl x509 -sha1 -fingerprint -in ca.crt -noout export EXP_CLUSTER_RESOURCE_SET="true" # This enables the ClusterResourceSet feature that we are using to deploy CSI export VSPHERE_SSH_AUTHORIZED_KEY="ssh-rsa AAAAB3N..." # The public ssh authorized key on all machines in this cluster. - # Set to "" if you don't want to enable SSH, or are using another solution. + # Set to "" if you don't want to enable SSH, or are using another solution. export VSPHERE_STORAGE_POLICY="" # This is the vSphere storage policy. Set it to "" if you don't want to use a storage policy. -export "CPI_IMAGE_K8S_VERSION"="v1.28.0" # The version of the vSphere CPI image to be used by the CPI workloads +export "CPI_IMAGE_K8S_VERSION"="v1.30.0" # The version of the vSphere CPI image to be used by the CPI workloads + +export CK8S_VERSION="v1.30.0" export VSPHERE_PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration From 2230b980c58d75984e6d3b8aedee0c966151c980 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 22 Jul 2024 19:55:58 -0400 Subject: [PATCH 20/30] add machine count vars --- templates/vsphere/cluster-template.yaml | 4 ++-- templates/vsphere/template-variables.rc | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index e8fe6320..60121aea 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -274,7 +274,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate name: ${CLUSTER_NAME} - replicas: 3 + replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: ${CK8S_VERSION} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 @@ -315,7 +315,7 @@ metadata: namespace: default spec: clusterName: ${CLUSTER_NAME} - replicas: 3 + replicas: ${WORKER_MACHINE_COUNT} selector: matchLabels: {} template: diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index 0c54368e..e8b41a77 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -1,5 +1,4 @@ -# Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md - +# <--- Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md export VSPHERE_USERNAME="" # The username used to access the remote vSphere endpoint export VSPHERE_PASSWORD="" # The password used to access the remote vSphere endpoint @@ -18,7 +17,11 @@ export VSPHERE_SSH_AUTHORIZED_KEY="ssh-rsa AAAAB3N..." # The public # Set to "" if you don't want to enable SSH, or are using another solution. export VSPHERE_STORAGE_POLICY="" # This is the vSphere storage policy. Set it to "" if you don't want to use a storage policy. export "CPI_IMAGE_K8S_VERSION"="v1.30.0" # The version of the vSphere CPI image to be used by the CPI workloads +# ---> + +export CK8S_VERSION="v1.30.0" # K8s Version of the cluster -export CK8S_VERSION="v1.30.0" +export VSPHERE_PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration -export VSPHERE_PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration +export CONTROL_PLANE_MACHINE_COUNT=3 +export WORKER_MACHINE_COUNT=1 \ No newline at end of file From 2dd8c9d774f9659681a42edbc7cd6a57ad99e450 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 22 Jul 2024 20:17:48 -0400 Subject: [PATCH 21/30] extract image variables --- templates/vsphere/cluster-template.yaml | 28 +++++++------- templates/vsphere/template-variables.rc | 51 +++++++++++++++---------- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 60121aea..62078702 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -776,7 +776,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 + image: ${CSI_ATTACHER_IMAGE} name: csi-attacher volumeMounts: - mountPath: /csi @@ -795,7 +795,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + image: ${CSI_RESIZER_IMAGE} name: csi-resizer volumeMounts: - mountPath: /csi @@ -824,7 +824,7 @@ data: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.3.0 + image: ${CSI_DRIVER_IMAGE} imagePullPolicy: Always livenessProbe: failureThreshold: 3 @@ -855,7 +855,7 @@ data: - args: - --v=4 - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: ${CSI_LIVENESS_PROBE_IMAGE} name: liveness-probe volumeMounts: - mountPath: /csi @@ -884,7 +884,7 @@ data: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.3.0 + image: ${CSI_SYNCER_IMAGE} imagePullPolicy: Always name: vsphere-syncer ports: @@ -913,7 +913,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + image: ${CSI_PROVISIONER_IMAGE} name: csi-provisioner volumeMounts: - mountPath: /csi @@ -931,7 +931,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + image: ${CSI_SNAPSHOTTER_IMAGE} name: csi-snapshotter volumeMounts: - mountPath: /csi @@ -979,7 +979,7 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + image: ${CSI_REGISTRAR_IMAGE} livenessProbe: exec: command: @@ -1021,7 +1021,7 @@ data: fieldPath: metadata.namespace - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES value: "1" - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.3.0 + image: ${CSI_DRIVER_IMAGE} imagePullPolicy: Always livenessProbe: failureThreshold: 3 @@ -1057,7 +1057,7 @@ data: - args: - --v=4 - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: ${CSI_LIVENESS_PROBE_IMAGE} name: liveness-probe volumeMounts: - mountPath: /csi @@ -1127,7 +1127,7 @@ data: value: unix://C:\\csi\\csi.sock - name: DRIVER_REG_SOCK_PATH value: C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + image: ${CSI_REGISTRAR_IMAGE} livenessProbe: exec: command: @@ -1170,7 +1170,7 @@ data: fieldPath: metadata.namespace - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES value: "1" - image: gcr.io/cloud-provider-vsphere/csi/release/driver:v3.1.0 + image: ${CSI_DRIVER_IMAGE} imagePullPolicy: Always livenessProbe: failureThreshold: 3 @@ -1201,7 +1201,7 @@ data: - args: - --v=4 - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: ${CSI_LIVENESS_PROBE_IMAGE} name: liveness-probe volumeMounts: - mountPath: /csi @@ -1431,7 +1431,7 @@ data: priorityClassName: system-node-critical containers: - name: vsphere-cpi - image: gcr.io/cloud-provider-vsphere/cpi/release/manager:${CPI_IMAGE_K8S_VERSION} + image: ${CPI_MANAGER_IMAGE} imagePullPolicy: IfNotPresent args: - --cloud-provider=vsphere diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index e8b41a77..f11f8f50 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -1,27 +1,36 @@ -# <--- Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md -export VSPHERE_USERNAME="" # The username used to access the remote vSphere endpoint -export VSPHERE_PASSWORD="" # The password used to access the remote vSphere endpoint +# Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md -export VSPHERE_SERVER="10.0.0.1" # The vCenter server IP or FQDN -export VSPHERE_DATACENTER="SDDC-Datacenter" # The vSphere datacenter to deploy the management cluster on -export VSPHERE_DATASTORE="DefaultDatastore" # The vSphere datastore to deploy the management cluster on -export VSPHERE_NETWORK="VM Network" # The VM network to deploy the management cluster on -export VSPHERE_RESOURCE_POOL="*/Resources" # The vSphere resource pool for your VMs -export VSPHERE_FOLDER="vm" # The VM folder for your VMs. Set to "" to use the root vSphere folder -export VSPHERE_TEMPLATE="ubuntu-1804-kube-v1.17.3" # The VM template to use for your management cluster. -export CONTROL_PLANE_ENDPOINT_IP="192.168.9.230" # the IP that kube-vip is going to use as a control plane endpoint -export VIP_NETWORK_INTERFACE="ens192" # The interface that kube-vip should apply the IP to. Omit to tell kube-vip to autodetect the interface. -export VSPHERE_TLS_THUMBPRINT="..." # sha1 thumbprint of the vcenter certificate: openssl x509 -sha1 -fingerprint -in ca.crt -noout -export EXP_CLUSTER_RESOURCE_SET="true" # This enables the ClusterResourceSet feature that we are using to deploy CSI -export VSPHERE_SSH_AUTHORIZED_KEY="ssh-rsa AAAAB3N..." # The public ssh authorized key on all machines in this cluster. - # Set to "" if you don't want to enable SSH, or are using another solution. -export VSPHERE_STORAGE_POLICY="" # This is the vSphere storage policy. Set it to "" if you don't want to use a storage policy. -export "CPI_IMAGE_K8S_VERSION"="v1.30.0" # The version of the vSphere CPI image to be used by the CPI workloads -# ---> +export VSPHERE_USERNAME="" # The username used to access the remote vSphere endpoint +export VSPHERE_PASSWORD="" # The password used to access the remote vSphere endpoint -export CK8S_VERSION="v1.30.0" # K8s Version of the cluster +export VSPHERE_SERVER="10.0.0.1" # The vCenter server IP or FQDN +export VSPHERE_DATACENTER="SDDC-Datacenter" # The vSphere datacenter to deploy the management cluster on +export VSPHERE_DATASTORE="DefaultDatastore" # The vSphere datastore to deploy the management cluster on +export VSPHERE_NETWORK="VM Network" # The VM network to deploy the management cluster on +export VSPHERE_RESOURCE_POOL="*/Resources" # The vSphere resource pool for your VMs +export VSPHERE_FOLDER="vm" # The VM folder for your VMs. Set to "" to use the root vSphere folder +export VSPHERE_TEMPLATE="ubuntu-1804-kube-v1.17.3" # The VM template to use for your management cluster. +export CONTROL_PLANE_ENDPOINT_IP="192.168.9.230" # the IP that kube-vip is going to use as a control plane endpoint +export VIP_NETWORK_INTERFACE="ens192" # The interface that kube-vip should apply the IP to. Omit to tell kube-vip to autodetect the interface. +export VSPHERE_TLS_THUMBPRINT="..." # sha1 thumbprint of the vcenter certificate: openssl x509 -sha1 -fingerprint -in ca.crt -noout +export EXP_CLUSTER_RESOURCE_SET="true" # This enables the ClusterResourceSet feature that we are using to deploy CSI +export VSPHERE_SSH_AUTHORIZED_KEY="ssh-rsa AAAAB3N..." # The public ssh authorized key on all machines in this cluster. + # Set to "" if you don't want to enable SSH, or are using another solution. +export VSPHERE_STORAGE_POLICY="" # This is the vSphere storage policy. Set it to "" if you don't want to use a storage policy. -export VSPHERE_PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration +export CPI_MANAGER_IMAGE="gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.30.0" # The version of the vSphere CPI image to be used by the CPI workloads +export CSI_DRIVER_IMAGE="gcr.io/cloud-provider-vsphere/csi/release/driver:v3.3.0" +export CSI_SYNCER_IMAGE="gcr.io/cloud-provider-vsphere/csi/release/syncer:v3.3.0" +export CSI_LIVENESS_PROBE_IMAGE="registry.k8s.io/sig-storage/livenessprobe:v2.10.0" +export CSI_ATTACHER_IMAGE="registry.k8s.io/sig-storage/csi-attacher:v4.3.0" +export CSI_RESIZER_IMAGE="registry.k8s.io/sig-storage/csi-resizer:v1.8.0" +export CSI_PROVISIONER_IMAGE="registry.k8s.io/sig-storage/csi-provisioner:v3.5.0" +export CSI_SNAPSHOTTER_IMAGE="registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2" +export CSI_REGISTRAR_IMAGE="registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0" + +export CK8S_VERSION="v1.30.0" # K8s Version of the cluster + +export VSPHERE_PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration export CONTROL_PLANE_MACHINE_COUNT=3 export WORKER_MACHINE_COUNT=1 \ No newline at end of file From b5d1b8548f3ca1dc672dd788a5f149cd05b7bb78 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Thu, 25 Jul 2024 08:37:51 -0400 Subject: [PATCH 22/30] review comments --- templates/vsphere/cluster-template.yaml | 94 ++++++++++++------------- 1 file changed, 45 insertions(+), 49 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 62078702..08449e5c 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -12,55 +12,12 @@ stringData: ${VSPHERE_PROXY_DISABLE}Environment="NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" ${VSPHERE_PROXY_DISABLE}Environment="no_proxy=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" --- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list","get","watch", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["list","get","watch", "update"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role -subjects: -- kind: ServiceAccount - name: kube-vip - namespace: kube-system ---- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} name: ${CLUSTER_NAME} - namespace: default spec: clusterNetwork: pods: @@ -151,7 +108,6 @@ apiVersion: controlplane.cluster.x-k8s.io/v1beta2 kind: CK8sControlPlane metadata: name: ${CLUSTER_NAME} - namespace: default spec: spec: controlPlane: @@ -164,6 +120,51 @@ spec: owner: root:root path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.conf permissions: "0644" + - content: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kube-vip + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role + rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:kube-vip-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role + subjects: + - kind: ServiceAccount + name: kube-vip + namespace: kube-system + owner: root:root + path: /capi/manifests/kube-vip-rbac.yaml + permissions: "0644" - content: | apiVersion: apps/v1 kind: DaemonSet @@ -281,7 +282,6 @@ apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate metadata: name: ${CLUSTER_NAME}-md-0 - namespace: default spec: template: spec: @@ -312,7 +312,6 @@ metadata: labels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} name: ${CLUSTER_NAME}-md-0 - namespace: default spec: clusterName: ${CLUSTER_NAME} replicas: ${WORKER_MACHINE_COUNT} @@ -341,7 +340,6 @@ metadata: labels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} name: ${CLUSTER_NAME}-crs-0 - namespace: default spec: clusterSelector: matchLabels: @@ -360,7 +358,6 @@ apiVersion: v1 kind: Secret metadata: name: ${CLUSTER_NAME} - namespace: default stringData: password: ${VSPHERE_PASSWORD} username: ${VSPHERE_USERNAME} @@ -369,7 +366,6 @@ apiVersion: v1 kind: Secret metadata: name: vsphere-config-secret - namespace: default stringData: data: | apiVersion: v1 From fa8845abef26dd2e803531fe7d41fd7fde5db9a0 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 5 Aug 2024 09:37:25 -0400 Subject: [PATCH 23/30] remove hardcoded namespaces --- templates/vsphere/cluster-template.yaml | 6 ------ templates/vsphere/template-variables.rc | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 08449e5c..b607fbd0 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -39,7 +39,6 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereCluster metadata: name: ${CLUSTER_NAME} - namespace: default spec: controlPlaneEndpoint: host: ${CONTROL_PLANE_ENDPOINT_IP} @@ -54,7 +53,6 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate metadata: name: ${CLUSTER_NAME} - namespace: default spec: template: spec: @@ -81,7 +79,6 @@ apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate metadata: name: ${CLUSTER_NAME}-worker - namespace: default spec: template: spec: @@ -1247,13 +1244,11 @@ data: kind: ConfigMap metadata: name: csi-manifests - namespace: default --- apiVersion: v1 kind: Secret metadata: name: cloud-provider-vsphere-credentials - namespace: default stringData: data: | apiVersion: v1 @@ -1511,4 +1506,3 @@ data: kind: ConfigMap metadata: name: cpi-manifests - namespace: default diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index f11f8f50..91aaf264 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -33,4 +33,4 @@ export CK8S_VERSION="v1.30.0" export VSPHERE_PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration export CONTROL_PLANE_MACHINE_COUNT=3 -export WORKER_MACHINE_COUNT=1 \ No newline at end of file +export WORKER_MACHINE_COUNT=1 From 53b9c82ec53169845041b971f430d1f777bcc01f Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 5 Aug 2024 12:25:50 -0400 Subject: [PATCH 24/30] cleanup variables --- templates/vsphere/cluster-template.yaml | 18 ++++++------ templates/vsphere/template-variables.rc | 38 ++++++++++++------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index b607fbd0..d106ce83 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -4,13 +4,13 @@ metadata: name: ${CLUSTER_NAME}-vsphere-proxy-config stringData: proxy-config: | - ${VSPHERE_PROXY_DISABLE}[Service] - ${VSPHERE_PROXY_DISABLE}Environment="HTTPS_PROXY=http://squid.internal:3128" - ${VSPHERE_PROXY_DISABLE}Environment="HTTP_PROXY=http://squid.internal:3128" - ${VSPHERE_PROXY_DISABLE}Environment="https_proxy=http://squid.internal:3128" - ${VSPHERE_PROXY_DISABLE}Environment="http_proxy=http://squid.internal:3128" - ${VSPHERE_PROXY_DISABLE}Environment="NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" - ${VSPHERE_PROXY_DISABLE}Environment="no_proxy=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" + ${PROXY_DISABLE}[Service] + ${PROXY_DISABLE}Environment="HTTPS_PROXY=http://squid.internal:3128" + ${PROXY_DISABLE}Environment="HTTP_PROXY=http://squid.internal:3128" + ${PROXY_DISABLE}Environment="https_proxy=http://squid.internal:3128" + ${PROXY_DISABLE}Environment="http_proxy=http://squid.internal:3128" + ${PROXY_DISABLE}Environment="NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" + ${PROXY_DISABLE}Environment="no_proxy=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster @@ -273,7 +273,7 @@ spec: kind: VSphereMachineTemplate name: ${CLUSTER_NAME} replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${CK8S_VERSION} + version: ${KUBERNETES_VERSION} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 kind: CK8sConfigTemplate @@ -329,7 +329,7 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate name: ${CLUSTER_NAME}-worker - version: ${CK8S_VERSION} + version: ${KUBERNETES_VERSION} --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index 91aaf264..fc3600c7 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -1,22 +1,22 @@ # Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md -export VSPHERE_USERNAME="" # The username used to access the remote vSphere endpoint -export VSPHERE_PASSWORD="" # The password used to access the remote vSphere endpoint +export VSPHERE_USERNAME="" # The username used to access the remote vSphere endpoint +export VSPHERE_PASSWORD="" # The password used to access the remote vSphere endpoint -export VSPHERE_SERVER="10.0.0.1" # The vCenter server IP or FQDN -export VSPHERE_DATACENTER="SDDC-Datacenter" # The vSphere datacenter to deploy the management cluster on -export VSPHERE_DATASTORE="DefaultDatastore" # The vSphere datastore to deploy the management cluster on -export VSPHERE_NETWORK="VM Network" # The VM network to deploy the management cluster on -export VSPHERE_RESOURCE_POOL="*/Resources" # The vSphere resource pool for your VMs -export VSPHERE_FOLDER="vm" # The VM folder for your VMs. Set to "" to use the root vSphere folder -export VSPHERE_TEMPLATE="ubuntu-1804-kube-v1.17.3" # The VM template to use for your management cluster. -export CONTROL_PLANE_ENDPOINT_IP="192.168.9.230" # the IP that kube-vip is going to use as a control plane endpoint -export VIP_NETWORK_INTERFACE="ens192" # The interface that kube-vip should apply the IP to. Omit to tell kube-vip to autodetect the interface. -export VSPHERE_TLS_THUMBPRINT="..." # sha1 thumbprint of the vcenter certificate: openssl x509 -sha1 -fingerprint -in ca.crt -noout -export EXP_CLUSTER_RESOURCE_SET="true" # This enables the ClusterResourceSet feature that we are using to deploy CSI -export VSPHERE_SSH_AUTHORIZED_KEY="ssh-rsa AAAAB3N..." # The public ssh authorized key on all machines in this cluster. - # Set to "" if you don't want to enable SSH, or are using another solution. -export VSPHERE_STORAGE_POLICY="" # This is the vSphere storage policy. Set it to "" if you don't want to use a storage policy. +export VSPHERE_SERVER="" # The vCenter server IP or FQDN, e.g. "10.0.0.1" +export VSPHERE_DATACENTER="" # The vSphere datacenter to deploy the management cluster on, e.g. "SDDC-Datacenter" +export VSPHERE_DATASTORE="" # The vSphere datastore to deploy the management cluster on, e.g. "DefaultDatastore" +export VSPHERE_NETWORK="" # The VM network to deploy the management cluster on, e.g. "VM Network" +export VSPHERE_RESOURCE_POOL="" # The vSphere resource pool for your VMs, e.g. "*/Resources" +export VSPHERE_FOLDER="" # The VM folder for your VMs. Set to "" to use the root vSphere folder, e.g. "vm" +export VSPHERE_TEMPLATE="" # The VM template to use for your management cluster, e.g. "ubuntu-1804-kube-v1.17.3" +export CONTROL_PLANE_ENDPOINT_IP="" # The IP that kube-vip is going to use as a control plane endpoint, e.g. "192.168.1.130" +export VIP_NETWORK_INTERFACE="" # The interface that kube-vip should apply the IP to. Omit to autodetect the interface, e.g. "ens192" +export VSPHERE_TLS_THUMBPRINT="" # SHA1 thumbprint of the vCenter certificate: openssl x509 -sha1 -fingerprint -in ca.crt -noout +export EXP_CLUSTER_RESOURCE_SET="true" # Enables the ClusterResourceSet feature used to deploy CSI +export VSPHERE_SSH_AUTHORIZED_KEY="" # The public SSH authorized key on all machines in this cluster. + # Set to "" if you don't want to enable SSH or are using another solution. +export VSPHERE_STORAGE_POLICY="" # The vSphere storage policy. Set to "" if you don't want to use a storage policy. export CPI_MANAGER_IMAGE="gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.30.0" # The version of the vSphere CPI image to be used by the CPI workloads export CSI_DRIVER_IMAGE="gcr.io/cloud-provider-vsphere/csi/release/driver:v3.3.0" @@ -28,9 +28,9 @@ export CSI_PROVISIONER_IMAGE="registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 export CSI_SNAPSHOTTER_IMAGE="registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2" export CSI_REGISTRAR_IMAGE="registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0" -export CK8S_VERSION="v1.30.0" # K8s Version of the cluster +export KUBERNETES_VERSION="" # K8s Version of the cluster, e.g. "v1.30.0" -export VSPHERE_PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration +export PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration -export CONTROL_PLANE_MACHINE_COUNT=3 +export CONTROL_PLANE_MACHINE_COUNT=1 export WORKER_MACHINE_COUNT=1 From 50543a56477ceddc8aad2f4d5a2c055fe98cc6c4 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 5 Aug 2024 12:54:32 -0400 Subject: [PATCH 25/30] ssh key optional --- templates/vsphere/cluster-template.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index d106ce83..05b2f85c 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -263,9 +263,12 @@ spec: >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts - - echo "${VSPHERE_SSH_AUTHORIZED_KEY}" >> /root/.ssh/authorized_keys - - chmod 700 /root/.ssh - - chmod 600 /root/.ssh/authorized_keys + - | + if [ -n "${VSPHERE_SSH_AUTHORIZED_KEY}" ]; then + echo "${VSPHERE_SSH_AUTHORIZED_KEY}" >> /root/.ssh/authorized_keys + chmod 700 /root/.ssh + chmod 600 /root/.ssh/authorized_keys + fi - snap install k8s --classic --edge machineTemplate: infrastructureTemplate: @@ -290,9 +293,12 @@ spec: >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts - - echo "${VSPHERE_SSH_AUTHORIZED_KEY}" >> /root/.ssh/authorized_keys - - chmod 700 /root/.ssh - - chmod 600 /root/.ssh/authorized_keys + - | + if [ -n "${VSPHERE_SSH_AUTHORIZED_KEY}" ]; then + echo "${VSPHERE_SSH_AUTHORIZED_KEY}" >> /root/.ssh/authorized_keys + chmod 700 /root/.ssh + chmod 600 /root/.ssh/authorized_keys + fi - snap install k8s --classic --edge files: - contentFrom: From dcfae77033619fdb4e44ddd8d2f1b34275caadeb Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Mon, 5 Aug 2024 12:55:39 -0400 Subject: [PATCH 26/30] indent --- templates/vsphere/template-variables.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index fc3600c7..8b2282a9 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -30,7 +30,7 @@ export CSI_REGISTRAR_IMAGE="registry.k8s.io/sig-storage/csi-node-driver-registra export KUBERNETES_VERSION="" # K8s Version of the cluster, e.g. "v1.30.0" -export PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration +export PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration export CONTROL_PLANE_MACHINE_COUNT=1 export WORKER_MACHINE_COUNT=1 From f88bb119205c8f6c14fc892ebf329fd3bf30aadf Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 13 Aug 2024 12:51:39 -0400 Subject: [PATCH 27/30] review comments --- templates/vsphere/cluster-template.yaml | 334 +++++++++++++----------- templates/vsphere/template-variables.rc | 25 +- 2 files changed, 198 insertions(+), 161 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 05b2f85c..76c4792e 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -1,16 +1,163 @@ apiVersion: v1 kind: Secret +metadata: + name: ${CLUSTER_NAME}-vsphere-proxy-env +stringData: + proxy-env: | + HTTP_PROXY=${HTTP_PROXY} + HTTPS_PROXY=${HTTPS_PROXY} + NO_PROXY=${NO_PROXY} +--- +apiVersion: v1 +kind: Secret metadata: name: ${CLUSTER_NAME}-vsphere-proxy-config stringData: proxy-config: | ${PROXY_DISABLE}[Service] - ${PROXY_DISABLE}Environment="HTTPS_PROXY=http://squid.internal:3128" - ${PROXY_DISABLE}Environment="HTTP_PROXY=http://squid.internal:3128" - ${PROXY_DISABLE}Environment="https_proxy=http://squid.internal:3128" - ${PROXY_DISABLE}Environment="http_proxy=http://squid.internal:3128" - ${PROXY_DISABLE}Environment="NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" - ${PROXY_DISABLE}Environment="no_proxy=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0" + ${PROXY_DISABLE}EnvironmentFile=/etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.env +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME}-kube-vip-rbac +stringData: + kube-vip-rbac: | + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kube-vip + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role + rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:kube-vip-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role + subjects: + - kind: ServiceAccount + name: kube-vip + namespace: kube-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: ${CLUSTER_NAME}-kube-vip-manifest +stringData: + kube-vip-manifest: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + selector: + matchLabels: + name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + name: kube-vip-ds + spec: + serviceAccountName: kube-vip + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: "${VIP_NETWORK_INTERFACE}" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: svc_election + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: vip_address + value: "${CONTROL_PLANE_ENDPOINT_IP}" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.4 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/hosts + name: etchosts + hostNetwork: true + volumes: + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} + status: {} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster @@ -110,6 +257,18 @@ spec: controlPlane: cloudProvider: external files: + - content: | + ${VSPHERE_SSH_AUTHORIZED_KEY} + owner: root:root + path: /root/.ssh/authorized_keys + permissions: "600" + - contentFrom: + secret: + key: proxy-env + name: ${CLUSTER_NAME}-vsphere-proxy-env + owner: root:root + path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.env + permissions: "0644" - contentFrom: secret: key: proxy-config @@ -117,139 +276,17 @@ spec: owner: root:root path: /etc/systemd/system/snap.k8s.containerd.service.d/http-proxy.conf permissions: "0644" - - content: | - apiVersion: v1 - kind: ServiceAccount - metadata: - name: kube-vip - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role - rules: - - apiGroups: [""] - resources: ["services/status"] - verbs: ["update"] - - apiGroups: [""] - resources: ["services", "endpoints"] - verbs: ["list","get","watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["list","get","watch", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] - - apiGroups: ["discovery.k8s.io"] - resources: ["endpointslices"] - verbs: ["list","get","watch", "update"] - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:kube-vip-binding - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role - subjects: - - kind: ServiceAccount - name: kube-vip - namespace: kube-system + - contentFrom: + secret: + key: kube-vip-rbac + name: ${CLUSTER_NAME}-kube-vip-rbac owner: root:root path: /capi/manifests/kube-vip-rbac.yaml permissions: "0644" - - content: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system - spec: - selector: - matchLabels: - name: kube-vip-ds - template: - metadata: - creationTimestamp: null - labels: - name: kube-vip-ds - spec: - serviceAccountName: kube-vip - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_interface - value: "${VIP_NETWORK_INTERFACE}" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: vip_ddns - value: "false" - - name: svc_enable - value: "true" - - name: svc_leasename - value: plndr-svcs-lock - - name: svc_election - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leasename - value: plndr-cp-lock - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: vip_address - value: "${CONTROL_PLANE_ENDPOINT_IP}" - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.6.4 - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/hosts - name: etchosts - hostNetwork: true - volumes: - - hostPath: - path: /etc/kube-vip.hosts - type: File - name: etchosts - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} - status: {} + - content: + secret: + key: kube-vip-manifest + name: ${CLUSTER_NAME}-kube-vip-manifest owner: root:root path: /capi/manifests/kube-vip.yaml permissions: "0644" @@ -259,17 +296,9 @@ spec: permissions: "0644" preRunCommands: - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" - >/etc/hosts + - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts - - | - if [ -n "${VSPHERE_SSH_AUTHORIZED_KEY}" ]; then - echo "${VSPHERE_SSH_AUTHORIZED_KEY}" >> /root/.ssh/authorized_keys - chmod 700 /root/.ssh - chmod 600 /root/.ssh/authorized_keys - fi - - snap install k8s --classic --edge machineTemplate: infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -289,18 +318,15 @@ spec: cloudProvider: external preRunCommands: - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" - >/etc/hosts + - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts - - | - if [ -n "${VSPHERE_SSH_AUTHORIZED_KEY}" ]; then - echo "${VSPHERE_SSH_AUTHORIZED_KEY}" >> /root/.ssh/authorized_keys - chmod 700 /root/.ssh - chmod 600 /root/.ssh/authorized_keys - fi - - snap install k8s --classic --edge files: + - content: | + ${VSPHERE_SSH_AUTHORIZED_KEY} + owner: root:root + path: /root/.ssh/authorized_keys + permissions: "600" - contentFrom: secret: key: proxy-config diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index 8b2282a9..3cb90bf6 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -1,3 +1,21 @@ +export KUBERNETES_VERSION="" # K8s Version of the cluster, e.g. "v1.30.0" + +export PROXY_DISABLE="" # Set to "#" to disable, or "" to enable the proxy configuration + # If enabled, make sure to configure the proxy in the following section. + +if [ "$PROXY_DISABLE" == "#" ]; then + export HTTP_PROXY="" + export HTTPS_PROXY="" + export NO_PROXY="" +else + export HTTP_PROXY=http://squid.internal:3128 + export HTTPS_PROXY=http://squid.internal:3128 + export NO_PROXY=10.0.0.0/8,192.168.0.0/16,127.0.0.1,localhost,0.0.0.0 +fi + +export CONTROL_PLANE_MACHINE_COUNT=1 +export WORKER_MACHINE_COUNT=1 + # Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md export VSPHERE_USERNAME="" # The username used to access the remote vSphere endpoint @@ -27,10 +45,3 @@ export CSI_RESIZER_IMAGE="registry.k8s.io/sig-storage/csi-resizer:v1.8.0" export CSI_PROVISIONER_IMAGE="registry.k8s.io/sig-storage/csi-provisioner:v3.5.0" export CSI_SNAPSHOTTER_IMAGE="registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2" export CSI_REGISTRAR_IMAGE="registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0" - -export KUBERNETES_VERSION="" # K8s Version of the cluster, e.g. "v1.30.0" - -export PROXY_DISABLE="#" # Set to "#" to disable, or "" to enable the proxy configuration - -export CONTROL_PLANE_MACHINE_COUNT=1 -export WORKER_MACHINE_COUNT=1 From 00bf2f1acdbf053300c3a762ecf2ebb7e66792f6 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 13 Aug 2024 15:31:32 -0400 Subject: [PATCH 28/30] content -> contentFrom --- templates/vsphere/cluster-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 76c4792e..92230d4f 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -283,7 +283,7 @@ spec: owner: root:root path: /capi/manifests/kube-vip-rbac.yaml permissions: "0644" - - content: + - contentFrom: secret: key: kube-vip-manifest name: ${CLUSTER_NAME}-kube-vip-manifest From a0f54c7b95badcbcbf062f5269512e6254290434 Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 13 Aug 2024 15:49:30 -0400 Subject: [PATCH 29/30] make machine specs configurable --- templates/vsphere/cluster-template.yaml | 12 ++++++------ templates/vsphere/template-variables.rc | 8 ++++++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/templates/vsphere/cluster-template.yaml b/templates/vsphere/cluster-template.yaml index 92230d4f..fc932d77 100644 --- a/templates/vsphere/cluster-template.yaml +++ b/templates/vsphere/cluster-template.yaml @@ -206,14 +206,14 @@ spec: cloneMode: linkedClone datacenter: ${VSPHERE_DATACENTER} datastore: ${VSPHERE_DATASTORE} - diskGiB: 25 + diskGiB: ${CONTROL_PLANE_MACHINE_DISK_GB} folder: ${VSPHERE_FOLDER} - memoryMiB: 8192 + memoryMiB: ${CONTROL_PLANE_MACHINE_MEMORY_MB} network: devices: - dhcp4: true networkName: ${VSPHERE_NETWORK} - numCPUs: 2 + numCPUs: ${CONTROL_PLANE_MACHINE_CPU_COUNT} os: Linux powerOffMode: trySoft resourcePool: '${VSPHERE_RESOURCE_POOL}' @@ -232,14 +232,14 @@ spec: cloneMode: linkedClone datacenter: ${VSPHERE_DATACENTER} datastore: ${VSPHERE_DATASTORE} - diskGiB: 25 + diskGiB: ${WORKER_MACHINE_DISK_GB} folder: ${VSPHERE_FOLDER} - memoryMiB: 8192 + memoryMiB: ${WORKER_MACHINE_MEMORY_MB} network: devices: - dhcp4: true networkName: ${VSPHERE_NETWORK} - numCPUs: 2 + numCPUs: ${WORKER_MACHINE_CPU_COUNT} os: Linux powerOffMode: trySoft resourcePool: '${VSPHERE_RESOURCE_POOL}' diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index 3cb90bf6..cf4dfd12 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -16,6 +16,14 @@ fi export CONTROL_PLANE_MACHINE_COUNT=1 export WORKER_MACHINE_COUNT=1 +export CONTROL_PLANE_MACHINE_CPU_COUNT=4 +export CONTROL_PLANE_MACHINE_MEMORY_MB=4096 +export CONTROL_PLANE_MACHINE_DISK_GB=25 + +export WORKER_MACHINE_CPU_COUNT=2 +export WORKER_MACHINE_MEMORY_MB=2048 +export WORKER_MACHINE_DISK_GB=25 + # Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md export VSPHERE_USERNAME="" # The username used to access the remote vSphere endpoint From 896db5075fc21ed7eda6b2d14bff4805664291d0 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Thu, 5 Sep 2024 10:03:34 -0400 Subject: [PATCH 30/30] remove empty str defaults --- templates/vsphere/template-variables.rc | 46 ++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/templates/vsphere/template-variables.rc b/templates/vsphere/template-variables.rc index cf4dfd12..e44feeaf 100644 --- a/templates/vsphere/template-variables.rc +++ b/templates/vsphere/template-variables.rc @@ -1,12 +1,12 @@ -export KUBERNETES_VERSION="" # K8s Version of the cluster, e.g. "v1.30.0" +export KUBERNETES_VERSION= # K8s Version of the cluster, e.g. "v1.30.0" -export PROXY_DISABLE="" # Set to "#" to disable, or "" to enable the proxy configuration - # If enabled, make sure to configure the proxy in the following section. +export PROXY_DISABLE= # Set to "#" to disable, or "" to enable the proxy configuration + # If enabled, make sure to configure the proxy in the following section. if [ "$PROXY_DISABLE" == "#" ]; then - export HTTP_PROXY="" - export HTTPS_PROXY="" - export NO_PROXY="" + export HTTP_PROXY= + export HTTPS_PROXY= + export NO_PROXY= else export HTTP_PROXY=http://squid.internal:3128 export HTTPS_PROXY=http://squid.internal:3128 @@ -26,23 +26,23 @@ export WORKER_MACHINE_DISK_GB=25 # Source: https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/docs/getting_started.md -export VSPHERE_USERNAME="" # The username used to access the remote vSphere endpoint -export VSPHERE_PASSWORD="" # The password used to access the remote vSphere endpoint - -export VSPHERE_SERVER="" # The vCenter server IP or FQDN, e.g. "10.0.0.1" -export VSPHERE_DATACENTER="" # The vSphere datacenter to deploy the management cluster on, e.g. "SDDC-Datacenter" -export VSPHERE_DATASTORE="" # The vSphere datastore to deploy the management cluster on, e.g. "DefaultDatastore" -export VSPHERE_NETWORK="" # The VM network to deploy the management cluster on, e.g. "VM Network" -export VSPHERE_RESOURCE_POOL="" # The vSphere resource pool for your VMs, e.g. "*/Resources" -export VSPHERE_FOLDER="" # The VM folder for your VMs. Set to "" to use the root vSphere folder, e.g. "vm" -export VSPHERE_TEMPLATE="" # The VM template to use for your management cluster, e.g. "ubuntu-1804-kube-v1.17.3" -export CONTROL_PLANE_ENDPOINT_IP="" # The IP that kube-vip is going to use as a control plane endpoint, e.g. "192.168.1.130" -export VIP_NETWORK_INTERFACE="" # The interface that kube-vip should apply the IP to. Omit to autodetect the interface, e.g. "ens192" -export VSPHERE_TLS_THUMBPRINT="" # SHA1 thumbprint of the vCenter certificate: openssl x509 -sha1 -fingerprint -in ca.crt -noout -export EXP_CLUSTER_RESOURCE_SET="true" # Enables the ClusterResourceSet feature used to deploy CSI -export VSPHERE_SSH_AUTHORIZED_KEY="" # The public SSH authorized key on all machines in this cluster. - # Set to "" if you don't want to enable SSH or are using another solution. -export VSPHERE_STORAGE_POLICY="" # The vSphere storage policy. Set to "" if you don't want to use a storage policy. +export VSPHERE_USERNAME= # The username used to access the remote vSphere endpoint +export VSPHERE_PASSWORD= # The password used to access the remote vSphere endpoint + +export VSPHERE_SERVER= # The vCenter server IP or FQDN, e.g. "10.0.0.1" +export VSPHERE_DATACENTER= # The vSphere datacenter to deploy the management cluster on, e.g. "SDDC-Datacenter" +export VSPHERE_DATASTORE= # The vSphere datastore to deploy the management cluster on, e.g. "DefaultDatastore" +export VSPHERE_NETWORK= # The VM network to deploy the management cluster on, e.g. "VM Network" +export VSPHERE_RESOURCE_POOL= # The vSphere resource pool for your VMs, e.g. "*/Resources" +export VSPHERE_FOLDER= # The VM folder for your VMs. Set to "" to use the root vSphere folder, e.g. "vm" +export VSPHERE_TEMPLATE= # The VM template to use for your management cluster, e.g. "ubuntu-1804-kube-v1.17.3" +export CONTROL_PLANE_ENDPOINT_IP= # The IP that kube-vip is going to use as a control plane endpoint, e.g. "192.168.1.130" +export VIP_NETWORK_INTERFACE= # The interface that kube-vip should apply the IP to. Omit to autodetect the interface, e.g. "ens192" +export VSPHERE_TLS_THUMBPRINT= # SHA1 thumbprint of the vCenter certificate: openssl x509 -sha1 -fingerprint -in ca.crt -noout +export EXP_CLUSTER_RESOURCE_SET="true" # Enables the ClusterResourceSet feature used to deploy CSI +export VSPHERE_SSH_AUTHORIZED_KEY= # The public SSH authorized key on all machines in this cluster. + # Set to "" if you don't want to enable SSH or are using another solution. +export VSPHERE_STORAGE_POLICY= # The vSphere storage policy. Set to "" if you don't want to use a storage policy. export CPI_MANAGER_IMAGE="gcr.io/cloud-provider-vsphere/cpi/release/manager:v1.30.0" # The version of the vSphere CPI image to be used by the CPI workloads export CSI_DRIVER_IMAGE="gcr.io/cloud-provider-vsphere/csi/release/driver:v3.3.0"