Add Registry Mirror config to Nutanix Control Plane Template

aws · Mar 13, 2023 · 0c3cc06 · 0c3cc06
1 parent 108c02e
commit 0c3cc06
Show file tree

Hide file tree

Showing 10 changed files with 750 additions and 172 deletions.
diff --git a/pkg/providers/nutanix/config/cp-template.yaml b/pkg/providers/nutanix/config/cp-template.yaml
@@ -88,80 +88,121 @@ spec:
           imageTag: {{.etcdImageTag}}
 {{- end }}
     files:
-      - content: |
-          apiVersion: v1
-          kind: Pod
-          metadata:
-            creationTimestamp: null
-            name: kube-vip
-            namespace: kube-system
-          spec:
-            containers:
-              - name: kube-vip
-                image: {{.kubeVipImage}}
-                imagePullPolicy: IfNotPresent
-                args:
-                  - manager
-                env:
-                  - name: vip_arp
-                    value: "true"
-                  - name: address
-                    value: "{{.controlPlaneEndpointIp}}"
-                  - name: port
-                    value: "6443"
-                  - name: vip_cidr
-                    value: "32"
-                  - name: cp_enable
-                    value: "true"
-                  - name: cp_namespace
-                    value: kube-system
-                  - name: vip_ddns
-                    value: "false"
-                  - name: vip_leaderelection
-                    value: "true"
-                  - name: vip_leaseduration
-                    value: "15"
-                  - name: vip_renewdeadline
-                    value: "10"
-                  - name: vip_retryperiod
-                    value: "2"
-                  - name: svc_enable
-                    value: "{{.kubeVipSvcEnable}}"
-                  - name: lb_enable
-                    value: "{{.kubeVipLBEnable}}"
-                securityContext:
-                  capabilities:
-                    add:
-                      - NET_ADMIN
-                      - SYS_TIME
-                      - NET_RAW
-                volumeMounts:
-                  - mountPath: /etc/kubernetes/admin.conf
-                    name: kubeconfig
-                resources: {}
-            hostNetwork: true
-            volumes:
-              - name: kubeconfig
-                hostPath:
-                  type: FileOrCreate
-                  path: /etc/kubernetes/admin.conf
-          status: {}
-        owner: root:root
-        path: /etc/kubernetes/manifests/kube-vip.yaml
+    - content: |
+        apiVersion: v1
+        kind: Pod
+        metadata:
+          creationTimestamp: null
+          name: kube-vip
+          namespace: kube-system
+        spec:
+          containers:
+            - name: kube-vip
+              image: {{.kubeVipImage}}
+              imagePullPolicy: IfNotPresent
+              args:
+                - manager
+              env:
+                - name: vip_arp
+                  value: "true"
+                - name: address
+                  value: "{{.controlPlaneEndpointIp}}"
+                - name: port
+                  value: "6443"
+                - name: vip_cidr
+                  value: "32"
+                - name: cp_enable
+                  value: "true"
+                - name: cp_namespace
+                  value: kube-system
+                - name: vip_ddns
+                  value: "false"
+                - name: vip_leaderelection
+                  value: "true"
+                - name: vip_leaseduration
+                  value: "15"
+                - name: vip_renewdeadline
+                  value: "10"
+                - name: vip_retryperiod
+                  value: "2"
+                - name: svc_enable
+                  value: "{{.kubeVipSvcEnable}}"
+                - name: lb_enable
+                  value: "{{.kubeVipLBEnable}}"
+              securityContext:
+                capabilities:
+                  add:
+                    - NET_ADMIN
+                    - SYS_TIME
+                    - NET_RAW
+              volumeMounts:
+                - mountPath: /etc/kubernetes/admin.conf
+                  name: kubeconfig
+              resources: {}
+          hostNetwork: true
+          volumes:
+            - name: kubeconfig
+              hostPath:
+                type: FileOrCreate
+                path: /etc/kubernetes/admin.conf
+        status: {}
+      owner: root:root
+      path: /etc/kubernetes/manifests/kube-vip.yaml
+{{- if .registryCACert }}
+    - content: |
+{{ .registryCACert | indent 8 }}
+      owner: root:root
+      path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+{{- end }}
+{{- if .registryMirrorMap }}
+    - content: |
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+          {{- range $orig, $mirror := .registryMirrorMap }}
+          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"]
+            endpoint = ["https://{{ $mirror }}"]
+          {{- end }}
+{{- if .registryCACert }}
+          [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls]
+            ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+{{- end }}
+{{- if .registryAuth }}
+          [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".auth]
+            username = "{{.registryUsername}}"
+            password = "{{.registryPassword}}"
+{{- end }}
+      owner: root:root
+      path: "/etc/containerd/config_append.toml"
+{{- end }}
     initConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
           # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
           # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
           #cgroup-driver: cgroupfs
           eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
+    joinConfiguration:
+      nodeRegistration:
+        criSocket: /var/run/containerd/containerd.sock
+        kubeletExtraArgs:
+          cloud-provider: external
+          read-only-port: "0"
+          anonymous-auth: "false"
+{{- if .kubeletExtraArgs }}
+{{ .kubeletExtraArgs.ToYaml | indent 10 }}
+{{- end }}
+        name: "{{`{{ ds.meta_data.hostname }}`}}"
     users:
       - name: "{{.controlPlaneSshUsername }}"
         lockPassword: false
         sudo: ALL=(ALL) NOPASSWD:ALL
         sshAuthorizedKeys:
           - "{{.controlPlaneSshAuthorizedKey}}"
     preKubeadmCommands:
+{{- if and .registryMirrorMap }}
+      - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml
+      - sudo systemctl daemon-reload
+      - sudo systemctl restart containerd
+{{- end }}
       - hostnamectl set-hostname "{{`{{ ds.meta_data.hostname }}`}}"
       - echo "::1         ipv6-localhost ipv6-loopback" >/etc/hosts
       - echo "127.0.0.1   localhost" >>/etc/hosts
@@ -211,3 +252,16 @@ spec:
         - type: uuid
           uuid: "{{.subnetUUID}}"
 {{ end }}
+---
+{{- if .registryAuth }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: registry-credentials
+  namespace: {{.eksaSystemNamespace}}
+  labels:
+    clusterctl.cluster.x-k8s.io/move: "true"
+stringData:
+  username: "{{.registryUsername}}"
+  password: "{{.registryPassword}}"
+{{- end }}
diff --git a/pkg/providers/nutanix/config/md-template.yaml b/pkg/providers/nutanix/config/md-template.yaml
@@ -79,6 +79,11 @@ spec:
   template:
     spec:
       preKubeadmCommands:
+{{- if .registryMirrorMap }}
+        - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml
+        - sudo systemctl daemon-reload
+        - sudo systemctl restart containerd
+{{- end }}
         - hostnamectl set-hostname "{{`{{ ds.meta_data.hostname }}`}}"
       joinConfiguration:
         nodeRegistration:
@@ -87,9 +92,41 @@ spec:
             # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
             #cgroup-driver: cgroupfs
             eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
+{{- if .kubeletExtraArgs }}
+{{ .kubeletExtraArgs.ToYaml | indent 12 }}
+{{- end }}
+          name: '{{`{{ ds.meta_data.hostname }}`}}'
       users:
         - name: "{{.workerSshUsername}}"
           lockPassword: false
           sudo: ALL=(ALL) NOPASSWD:ALL
           sshAuthorizedKeys:
             - "{{.workerSshAuthorizedKey}}"
+{{- if .registryMirrorMap }}
+      files:
+{{- end }}
+{{- if .registryCACert }}
+      - content: |
+{{ .registryCACert | indent 10 }}
+        owner: root:root
+        path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+{{- end }}
+{{- if .registryMirrorMap }}
+      - content: |
+          [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+            {{- range $orig, $mirror := .registryMirrorMap }}
+            [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"]
+              endpoint = ["https://{{ $mirror }}"]
+{{- end }}
+{{- if .registryCACert }}
+            [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls]
+              ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+{{- end }}
+{{- if .registryAuth }}
+            [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".auth]
+              username = "{{.registryUsername}}"
+              password = "{{.registryPassword}}"
+{{- end }}
+        owner: root:root
+        path: "/etc/containerd/config_append.toml"
+{{- end }}
diff --git a/pkg/providers/nutanix/template.go b/pkg/providers/nutanix/template.go
@@ -10,9 +10,12 @@ import (
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
 	"github.com/aws/eks-anywhere/pkg/cluster"
 	"github.com/aws/eks-anywhere/pkg/clusterapi"
+	"github.com/aws/eks-anywhere/pkg/config"
 	"github.com/aws/eks-anywhere/pkg/constants"
 	"github.com/aws/eks-anywhere/pkg/crypto"
 	"github.com/aws/eks-anywhere/pkg/providers"
+	"github.com/aws/eks-anywhere/pkg/registrymirror"
+	"github.com/aws/eks-anywhere/pkg/registrymirror/containerd"
 	"github.com/aws/eks-anywhere/pkg/templater"
 	"github.com/aws/eks-anywhere/pkg/types"
 )
@@ -55,7 +58,10 @@ func (ntb *TemplateBuilder) GenerateCAPISpecControlPlane(clusterSpec *cluster.Sp
 		etcdMachineSpec = *ntb.etcdMachineSpec
 	}
 
-	values := buildTemplateMapCP(ntb.datacenterSpec, clusterSpec, *ntb.controlPlaneMachineSpec, etcdMachineSpec)
+	values, err := buildTemplateMapCP(ntb.datacenterSpec, clusterSpec, *ntb.controlPlaneMachineSpec, etcdMachineSpec)
+	if err != nil {
+		return nil, err
+	}
 	for _, buildOption := range buildOptions {
 		buildOption(values)
 	}
@@ -71,7 +77,10 @@ func (ntb *TemplateBuilder) GenerateCAPISpecControlPlane(clusterSpec *cluster.Sp
 func (ntb *TemplateBuilder) GenerateCAPISpecWorkers(clusterSpec *cluster.Spec, workloadTemplateNames, kubeadmconfigTemplateNames map[string]string) (content []byte, err error) {
 	workerSpecs := make([][]byte, 0, len(clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations))
 	for _, workerNodeGroupConfiguration := range clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations {
-		values := buildTemplateMapMD(clusterSpec, ntb.workerNodeGroupMachineSpecs[workerNodeGroupConfiguration.MachineGroupRef.Name], workerNodeGroupConfiguration)
+		values, err := buildTemplateMapMD(clusterSpec, ntb.workerNodeGroupMachineSpecs[workerNodeGroupConfiguration.MachineGroupRef.Name], workerNodeGroupConfiguration)
+		if err != nil {
+			return nil, err
+		}
 		values["workloadTemplateName"] = workloadTemplateNames[workerNodeGroupConfiguration.Name]
 		values["workloadkubeadmconfigTemplateName"] = kubeadmconfigTemplateNames[workerNodeGroupConfiguration.Name]
 		values["autoscalingConfig"] = workerNodeGroupConfiguration.AutoScalingConfiguration
@@ -133,11 +142,14 @@ func buildTemplateMapCP(
 	clusterSpec *cluster.Spec,
 	controlPlaneMachineSpec v1alpha1.NutanixMachineConfigSpec,
 	etcdMachineSpec v1alpha1.NutanixMachineConfigSpec,
-) map[string]interface{} {
+) (map[string]interface{}, error) {
 	bundle := clusterSpec.VersionsBundle
 	format := "cloud-config"
 	apiServerExtraArgs := clusterapi.OIDCToExtraArgs(clusterSpec.OIDCConfig)
 
+	kubeletExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
+		Append(clusterapi.ResolvConfExtraArgs(clusterSpec.Cluster.Spec.ClusterNetwork.DNS.ResolvConf)).
+		Append(clusterapi.ControlPlaneNodeLabelsExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration))
 	values := map[string]interface{}{
 		"apiServerExtraArgs":           apiServerExtraArgs.ToPartialYaml(),
 		"clusterName":                  clusterSpec.Cluster.Name,
@@ -155,6 +167,7 @@ func buildTemplateMapCP(
 		"corednsVersion":               bundle.KubeDistro.CoreDNS.Tag,
 		"etcdRepository":               bundle.KubeDistro.Etcd.Repository,
 		"etcdImageTag":                 bundle.KubeDistro.Etcd.Tag,
+		"kubeletExtraArgs":             kubeletExtraArgs.ToPartialYaml(),
 		"kubeVipImage":                 bundle.Nutanix.KubeVip.VersionedImage(),
 		"kubeVipSvcEnable":             false,
 		"kubeVipLBEnable":              false,
@@ -180,19 +193,42 @@ func buildTemplateMapCP(
 		"subnetUUID":                   controlPlaneMachineSpec.Subnet.UUID,
 	}
 
+	if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil {
+		registryMirror := registrymirror.FromCluster(clusterSpec.Cluster)
+		values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap)
+		values["mirrorBase"] = registryMirror.BaseRegistry
+		values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror())
+		if len(registryMirror.CACertContent) > 0 {
+			values["registryCACert"] = registryMirror.CACertContent
+		}
+
+		if registryMirror.Auth {
+			values["registryAuth"] = registryMirror.Auth
+			username, password, err := config.ReadCredentials()
+			if err != nil {
+				return values, err
+			}
+			values["registryUsername"] = username
+			values["registryPassword"] = password
+		}
+	}
+
 	if clusterSpec.Cluster.Spec.ExternalEtcdConfiguration != nil {
 		values["externalEtcd"] = true
 		values["externalEtcdReplicas"] = clusterSpec.Cluster.Spec.ExternalEtcdConfiguration.Count
 		values["etcdSshUsername"] = etcdMachineSpec.Users[0].Name
 	}
 
-	return values
+	return values, nil
 }
 
-func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupMachineSpec v1alpha1.NutanixMachineConfigSpec, workerNodeGroupConfiguration v1alpha1.WorkerNodeGroupConfiguration) map[string]interface{} {
+func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupMachineSpec v1alpha1.NutanixMachineConfigSpec, workerNodeGroupConfiguration v1alpha1.WorkerNodeGroupConfiguration) (map[string]interface{}, error) {
 	bundle := clusterSpec.VersionsBundle
 	format := "cloud-config"
 
+	kubeletExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
+		Append(clusterapi.ResolvConfExtraArgs(clusterSpec.Cluster.Spec.ClusterNetwork.DNS.ResolvConf)).
+		Append(clusterapi.WorkerNodeLabelsExtraArgs(workerNodeGroupConfiguration))
 	values := map[string]interface{}{
 		"clusterName":            clusterSpec.Cluster.Name,
 		"eksaSystemNamespace":    constants.EksaSystemNamespace,
@@ -209,6 +245,7 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupMachineSpec v1
 		"imageIDType":            workerNodeGroupMachineSpec.Image.Type,
 		"imageName":              workerNodeGroupMachineSpec.Image.Name,
 		"imageUUID":              workerNodeGroupMachineSpec.Image.UUID,
+		"kubeletExtraArgs":       kubeletExtraArgs.ToPartialYaml(),
 		"nutanixPEClusterIDType": workerNodeGroupMachineSpec.Cluster.Type,
 		"nutanixPEClusterName":   workerNodeGroupMachineSpec.Cluster.Name,
 		"nutanixPEClusterUUID":   workerNodeGroupMachineSpec.Cluster.UUID,
@@ -217,7 +254,28 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupMachineSpec v1
 		"subnetUUID":             workerNodeGroupMachineSpec.Subnet.UUID,
 		"workerNodeGroupName":    fmt.Sprintf("%s-%s", clusterSpec.Cluster.Name, workerNodeGroupConfiguration.Name),
 	}
-	return values
+
+	if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil {
+		registryMirror := registrymirror.FromCluster(clusterSpec.Cluster)
+		values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap)
+		values["mirrorBase"] = registryMirror.BaseRegistry
+		values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror())
+		if len(registryMirror.CACertContent) > 0 {
+			values["registryCACert"] = registryMirror.CACertContent
+		}
+
+		if registryMirror.Auth {
+			values["registryAuth"] = registryMirror.Auth
+			username, password, err := config.ReadCredentials()
+			if err != nil {
+				return values, err
+			}
+			values["registryUsername"] = username
+			values["registryPassword"] = password
+		}
+	}
+
+	return values, nil
 }
 
 func buildTemplateMapSecret(secretName string, creds credentials.BasicAuthCredential) (map[string]interface{}, error) {