From ae33f42017413367281cf1ba2ed5cac138e8fa4d Mon Sep 17 00:00:00 2001 From: vardhaman Date: Wed, 18 Dec 2024 10:18:47 +0530 Subject: [PATCH 1/2] fix service account issuer for custom cluster domain --- pkg/k8s/k8s.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index 21309b042..e8adce379 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -97,6 +97,12 @@ func StartK8S( if apiServer.Enabled { eg.Go(func() error { // build flags + issuer := "https://kubernetes.default.svc.cluster.local" + + if vConfig.Networking.Advanced.ClusterDomain != "" { + issuer = "https://kubernetes.default.svc." + vConfig.Networking.Advanced.ClusterDomain + } + args := []string{} if len(apiServer.Command) > 0 { args = append(args, apiServer.Command...) @@ -123,7 +129,7 @@ func StartK8S( args = append(args, "--requestheader-group-headers=X-Remote-Group") args = append(args, "--requestheader-username-headers=X-Remote-User") args = append(args, "--secure-port=6443") - args = append(args, "--service-account-issuer=https://kubernetes.default.svc.cluster.local") + args = append(args, "--service-account-issuer="+issuer) args = append(args, "--service-account-key-file=/data/pki/sa.pub") args = append(args, "--service-account-signing-key-file=/data/pki/sa.key") args = append(args, "--tls-cert-file=/data/pki/apiserver.crt") From 2ed229732695cd016757aa2d80d97d65d120c5f7 Mon Sep 17 00:00:00 2001 From: vardhaman Date: Mon, 16 Dec 2024 19:51:13 +0530 Subject: [PATCH 2/2] set clusterdomain value in _coredns.tpl --- chart/templates/_coredns.tpl | 2 +- chart/tests/coredns-configmap_test.yaml | 428 ++++++++++++++++++++++++ 2 files changed, 429 insertions(+), 1 deletion(-) diff --git a/chart/templates/_coredns.tpl b/chart/templates/_coredns.tpl index 8ecd7c047..945cc197c 100644 --- a/chart/templates/_coredns.tpl +++ b/chart/templates/_coredns.tpl @@ -16,7 +16,7 @@ Corefile: |- {{- if .Values.networking.advanced.proxyKubelets.byHostname }} rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local {{- end }} - kubernetes cluster.local in-addr.arpa ip6.arpa { + kubernetes{{ if and (.Values.networking.advanced.clusterDomain) (ne .Values.networking.advanced.clusterDomain "cluster.local") }} {{ .Values.networking.advanced.clusterDomain }}{{ end }} cluster.local in-addr.arpa ip6.arpa { {{- if .Values.controlPlane.coredns.embedded }} kubeconfig /data/vcluster/admin.conf {{- end }} diff --git a/chart/tests/coredns-configmap_test.yaml b/chart/tests/coredns-configmap_test.yaml index 88efcc4af..095e926e1 100644 --- a/chart/tests/coredns-configmap_test.yaml +++ b/chart/tests/coredns-configmap_test.yaml @@ -575,3 +575,431 @@ tests: - name: metrics port: 9153 protocol: TCP + - it: should correctly set default coredns clusterdomain + set: + networking: + advanced: + clusterDomain: cluster.local + asserts: + - equal: + path: data["coredns.yaml"] + value: |- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: coredns + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns + subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + Corefile: |- + .:1053 { + errors + health + ready + rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + hosts /etc/NodeHosts { + ttl 60 + reload 15s + fallthrough + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + loadbalance + } + + import /etc/coredns/custom/*.server + NodeHosts: "" + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: vcluster-kube-dns + kubernetes.io/name: "CoreDNS" + spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: vcluster-kube-dns + template: + metadata: + labels: + k8s-app: vcluster-kube-dns + spec: + priorityClassName: "" + serviceAccountName: coredns + nodeSelector: + kubernetes.io/os: linux + topologySpreadConstraints: + - labelSelector: + matchLabels: + k8s-app: vcluster-kube-dns + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + containers: + - name: coredns + image: {{.IMAGE}} + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 1000m + memory: 170Mi + requests: + cpu: 20m + memory: 64Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + - name: custom-config-volume + mountPath: /etc/coredns/custom + readOnly: true + securityContext: + runAsNonRoot: true + runAsUser: {{.RUN_AS_USER}} + runAsGroup: {{.RUN_AS_GROUP}} + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 2 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + - key: NodeHosts + path: NodeHosts + - name: custom-config-volume + configMap: + name: coredns-custom + optional: true + --- + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: vcluster-kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + spec: + type: ClusterIP + selector: + k8s-app: vcluster-kube-dns + ports: + - name: dns + port: 53 + targetPort: 1053 + protocol: UDP + - name: dns-tcp + port: 53 + targetPort: 1053 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + - it: should correctly set custom coredns clusterdomain + set: + networking: + advanced: + clusterDomain: custom.local + asserts: + - equal: + path: data["coredns.yaml"] + value: |- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: coredns + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns + subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + Corefile: |- + .:1053 { + errors + health + ready + rewrite name regex .*\.nodes\.vcluster\.com kubernetes.default.svc.cluster.local + kubernetes custom.local cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + hosts /etc/NodeHosts { + ttl 60 + reload 15s + fallthrough + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + loadbalance + } + + import /etc/coredns/custom/*.server + NodeHosts: "" + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: vcluster-kube-dns + kubernetes.io/name: "CoreDNS" + spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: vcluster-kube-dns + template: + metadata: + labels: + k8s-app: vcluster-kube-dns + spec: + priorityClassName: "" + serviceAccountName: coredns + nodeSelector: + kubernetes.io/os: linux + topologySpreadConstraints: + - labelSelector: + matchLabels: + k8s-app: vcluster-kube-dns + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + containers: + - name: coredns + image: {{.IMAGE}} + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 1000m + memory: 170Mi + requests: + cpu: 20m + memory: 64Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + - name: custom-config-volume + mountPath: /etc/coredns/custom + readOnly: true + securityContext: + runAsNonRoot: true + runAsUser: {{.RUN_AS_USER}} + runAsGroup: {{.RUN_AS_GROUP}} + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 2 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + - key: NodeHosts + path: NodeHosts + - name: custom-config-volume + configMap: + name: coredns-custom + optional: true + --- + apiVersion: v1 + kind: Service + metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: vcluster-kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" + spec: + type: ClusterIP + selector: + k8s-app: vcluster-kube-dns + ports: + - name: dns + port: 53 + targetPort: 1053 + protocol: UDP + - name: dns-tcp + port: 53 + targetPort: 1053 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP