Merge pull request #242 from openinfradev/capa_fix_external_cloud_pro…

…vider cluster-api-aws: fix for external cloud contoller mananger
openinfradev · Aug 23, 2024 · 71c8d62 · 71c8d62
2 parents 6fb2b7c + 3e8de84
commit 71c8d62
Show file tree

Hide file tree

Showing 4 changed files with 205 additions and 4 deletions.
diff --git a/cluster-api-aws/templates/crs-aws-ccm.yaml b/cluster-api-aws/templates/crs-aws-ccm.yaml
@@ -0,0 +1,197 @@
+apiVersion: addons.cluster.x-k8s.io/v1beta1
+kind: ClusterResourceSet
+metadata:
+  name: crs-ccm
+  namespace: {{ .Release.Namespace }}
+spec:
+  clusterSelector:
+    matchLabels:
+      ccm: external
+  resources:
+  - kind: ConfigMap
+    name: cloud-controller-manager-addon
+  strategy: ApplyOnce
+---
+apiVersion: v1
+data:
+  aws-ccm-external.yaml: |
+    ---
+    apiVersion: apps/v1
+    kind: DaemonSet
+    metadata:
+      name: aws-cloud-controller-manager
+      namespace: kube-system
+      labels:
+        k8s-app: aws-cloud-controller-manager
+    spec:
+      selector:
+        matchLabels:
+          k8s-app: aws-cloud-controller-manager
+      updateStrategy:
+        type: RollingUpdate
+      template:
+        metadata:
+          labels:
+            k8s-app: aws-cloud-controller-manager
+        spec:
+          nodeSelector:
+            node-role.kubernetes.io/control-plane: ""
+          tolerations:
+            - key: node.cloudprovider.kubernetes.io/uninitialized
+              value: "true"
+              effect: NoSchedule
+            - key: node-role.kubernetes.io/control-plane
+              effect: NoSchedule
+            - key: node.kubernetes.io/not-ready
+              effect: NoSchedule
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                      - key: node-role.kubernetes.io/control-plane
+                        operator: Exists
+          serviceAccountName: cloud-controller-manager
+          containers:
+            - name: aws-cloud-controller-manager
+              image: {{ .Values.awsCloudControllerManager.image.repository }}:{{ .Values.awsCloudControllerManager.image.tag }}
+              args:
+                - --v=2
+                - --cloud-provider=aws
+                - --use-service-account-credentials=true
+                - --configure-cloud-routes=false
+              resources:
+                requests:
+                  cpu: 200m
+          hostNetwork: true
+    ---
+    apiVersion: v1
+    kind: ServiceAccount
+    metadata:
+      name: cloud-controller-manager
+      namespace: kube-system
+    ---
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata:
+      name: cloud-controller-manager:apiserver-authentication-reader
+      namespace: kube-system
+    roleRef:
+      apiGroup: rbac.authorization.k8s.io
+      kind: Role
+      name: extension-apiserver-authentication-reader
+    subjects:
+      - apiGroup: ""
+        kind: ServiceAccount
+        name: cloud-controller-manager
+        namespace: kube-system
+    ---
+    apiVersion: rbac.authorization.k8s.io/v1
+    kind: ClusterRole
+    metadata:
+      name: system:cloud-controller-manager
+    rules:
+    - apiGroups:
+      - ""
+      resources:
+      - events
+      verbs:
+      - create
+      - patch
+      - update
+    - apiGroups:
+      - ""
+      resources:
+      - nodes
+      verbs:
+      - '*'
+    - apiGroups:
+      - ""
+      resources:
+      - nodes/status
+      verbs:
+      - patch
+    - apiGroups:
+      - ""
+      resources:
+      - services
+      verbs:
+      - list
+      - patch
+      - update
+      - watch
+    - apiGroups:
+      - ""
+      resources:
+      - services/status
+      verbs:
+      - list
+      - patch
+      - update
+      - watch
+    - apiGroups:
+      - ""
+      resources:
+      - serviceaccounts
+      verbs:
+      - create
+      - get
+      - list
+      - watch
+    - apiGroups:
+      - ""
+      resources:
+      - persistentvolumes
+      verbs:
+      - get
+      - list
+      - update
+      - watch
+    - apiGroups:
+      - ""
+      resources:
+      - endpoints
+      verbs:
+      - create
+      - get
+      - list
+      - watch
+      - update
+    - apiGroups:
+      - coordination.k8s.io
+      resources:
+      - leases
+      verbs:
+      - create
+      - get
+      - list
+      - watch
+      - update
+    - apiGroups:
+      - ""
+      resources:
+      - serviceaccounts/token
+      verbs:
+      - create
+    ---
+    kind: ClusterRoleBinding
+    apiVersion: rbac.authorization.k8s.io/v1
+    metadata:
+      name: system:cloud-controller-manager
+    roleRef:
+      apiGroup: rbac.authorization.k8s.io
+      kind: ClusterRole
+      name: system:cloud-controller-manager
+    subjects:
+      - apiGroup: ""
+        kind: ServiceAccount
+        name: cloud-controller-manager
+        namespace: kube-system
+kind: ConfigMap
+metadata:
+  annotations:
+    note: generated
+  labels:
+    type: generated
+  name: cloud-controller-manager-addon
+  namespace: {{ .Release.Namespace }}
diff --git a/cluster-api-aws/templates/machine-deployment.yaml b/cluster-api-aws/templates/machine-deployment.yaml
@@ -42,7 +42,7 @@ spec:
       clusterName: {{ $envAll.Values.cluster.name }}
       failureDomain: {{ $envAll.Values.cluster.region }}{{ index $azPostfixList . }}
       infrastructureRef:
-        apiVersion: {{ $envAll.Values.api.group.infrastructure }}/{{ $envAll.Values.api.version }}
+        apiVersion: {{ $envAll.Values.api.group.infrastructure }}/v1beta2
         kind: AWSMachineTemplate
         name: {{ $envAll.Values.cluster.name }}-md-{{ $mdName }}
       version: {{ $envAll.Values.cluster.kubernetesVersion }}
@@ -53,7 +53,7 @@ spec:
 ---
 {{- end }}
 ---
-apiVersion: {{ $envAll.Values.api.group.infrastructure }}/{{ $envAll.Values.api.version }}
+apiVersion: {{ $envAll.Values.api.group.infrastructure }}/v1beta2
 kind: AWSMachineTemplate
 metadata:
   name: {{ $envAll.Values.cluster.name }}-md-{{ $mdName }}
@@ -106,7 +106,7 @@ spec:
         nodeRegistration:
           {{- with $envAll.Values.kubeadmConfig.kubeletExtraArgs }}
           kubeletExtraArgs:
-            cloud-provider: aws
+            cloud-provider: external
             {{ toYaml . }}
           name: '{{`{{ ds.meta_data.local_hostname }}`}}'
           {{- end }}

diff --git a/cluster-api-aws/templates/machine-pool.yaml b/cluster-api-aws/templates/machine-pool.yaml
@@ -103,7 +103,7 @@ spec:
     nodeRegistration:
       {{- with $envAll.Values.kubeadmConfig.kubeletExtraArgs }}
       kubeletExtraArgs:
-        cloud-provider: aws
+        cloud-provider: external
         {{ toYaml . }}
       name: '{{`{{ ds.meta_data.local_hostname }}`}}'
       {{- end }}

diff --git a/cluster-api-aws/values.yaml b/cluster-api-aws/values.yaml
@@ -127,6 +127,10 @@ machineDeployment: []
 #     # MaxPrice defines the maximum price the user is willing to pay for Spot VM instances
 #   additionalSecurityGroups: []
 
+awsCloudControllerManager:
+  image:
+    repository: registry.k8s.io/provider-aws/cloud-controller-manager
+    tag: v1.28.3
 nameOverride: ""
 fullnameOverride: ""