Skip to content

Commit

Permalink
feat(helm): add possibility to run universal zone cp on kubernetes (#…
Browse files Browse the repository at this point in the history
…7924)

* feat(helm): Add possibility to run universal zone cp on kubernetes

Signed-off-by: Marcin Skalski <marcin.skalski@konghq.com>
  • Loading branch information
Automaat authored Oct 5, 2023
1 parent d756e47 commit 671b992
Show file tree
Hide file tree
Showing 8 changed files with 492 additions and 48 deletions.
29 changes: 20 additions & 9 deletions app/kumactl/cmd/install/install_control_plane_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,19 @@ var _ = Describe("kumactl install control-plane", func() {
},
goldenFile: "install-control-plane.global-universal-on-k8s.golden.yaml",
}),
Entry("should generate Kubernetes resources for Zone Universal mode", testCase{
extraArgs: []string{
"--mode",
"zone",
"--set",
"controlPlane.environment=universal",
"--kds-global-address",
"grpcs://192.168.0.1:5685",
"--zone",
"zone-1",
},
goldenFile: "install-control-plane.zone-universal-on-k8s.golden.yaml",
}),
Entry("should generate Kubernetes resources for Zone", testCase{
extraArgs: []string{
"--mode", "zone",
Expand Down Expand Up @@ -341,15 +354,13 @@ controlPlane:
extraArgs: []string{"--mode", "test"},
errorMsg: "controlPlane.mode invalid got:'test'",
}),
Entry("--mode is not global and environment is universal", errTestCase{
extraArgs: []string{
"--mode",
"zone",
"--set",
"controlPlane.environment=universal",
},
errorMsg: "Currently you can only run universal mode on kubernetes in a global mode, " +
"this limitation might be lifted in the future",
Entry("", errTestCase{
extraArgs: []string{"--kds-global-address", "grpcs://192.168.0.1:5685", "--mode", "zone", "--zone", "zone-1", "--set", "controlPlane.environment=universal", "--set", "egress.enabled=true"},
errorMsg: "Can't have egress.enabled when running controlPlane.mode=='universal'",
}),
Entry("", errTestCase{
extraArgs: []string{"--kds-global-address", "grpcs://192.168.0.1:5685", "--mode", "zone", "--zone", "zone-1", "--set", "controlPlane.environment=universal", "--set", "egress.enabled=true"},
errorMsg: "Can't have egress.enabled when running controlPlane.mode=='universal'",
}),
Entry("--kds-global-address is missing when installing zone", errTestCase{
extraArgs: []string{"--mode", "zone", "--zone", "zone-1"},
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,258 @@

---
apiVersion: v1
kind: Namespace
metadata:
name: kuma-system
labels:
kuma.io/system-namespace: "true"
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: kuma-control-plane
namespace: kuma-system
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kuma-control-plane-config
namespace: kuma-system
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
data:
config.yaml: |
# use this file to override default configuration of `kuma-cp`
#
# see conf/kuma-cp.conf.yml for available settings
---
apiVersion: v1
kind: Service
metadata:
name: kuma-control-plane
namespace: kuma-system
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5680"
spec:
type: ClusterIP
ports:
- port: 5680
name: diagnostics
appProtocol: http
- port: 5681
name: http-api-server
appProtocol: http
- port: 5682
name: https-api-server
appProtocol: https
- port: 5676
name: mads-server
appProtocol: https
- port: 5678
name: dp-server
appProtocol: https
selector:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kuma-control-plane
namespace: kuma-system
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
spec:
replicas: 1
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
selector:
matchLabels:
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
app: kuma-control-plane
template:
metadata:
annotations:
checksum/config: fd9d1d8386f97f2bd49e50f476520816168a1c9f60bbc43dec1347a64d239155
checksum/tls-secrets: 75a11da44c802486bc6f65640aa48a730f0f684c5c07a42ba3cd1735eb3fb070
labels:
app: kuma-control-plane
app.kubernetes.io/name: kuma
app.kubernetes.io/instance: kuma
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values:
- 'kuma'
- key: app.kubernetes.io/instance
operator: In
values:
- 'kuma'
- key: app
operator: In
values:
- 'kuma-control-plane'
topologyKey: kubernetes.io/hostname
weight: 100
securityContext:
runAsNonRoot: true
serviceAccountName: kuma-control-plane
automountServiceAccountToken: true
nodeSelector:

kubernetes.io/os: linux
hostNetwork: false
terminationGracePeriodSeconds: 30
initContainers:
- name: migration
image: "docker.io/kumahq/kuma-cp:0.0.1"
imagePullPolicy: IfNotPresent
securityContext:
readOnlyRootFilesystem: true
env:
- name: KUMA_DEFAULTS_SKIP_MESH_CREATION
value: "false"
- name: KUMA_ENVIRONMENT
value: "universal"
- name: KUMA_GENERAL_TLS_CERT_FILE
value: "/var/run/secrets/kuma.io/tls-cert/tls.crt"
- name: KUMA_GENERAL_TLS_KEY_FILE
value: "/var/run/secrets/kuma.io/tls-cert/tls.key"
- name: KUMA_GENERAL_WORK_DIR
value: "/tmp/kuma"
- name: KUMA_MODE
value: "zone"
- name: KUMA_MULTIZONE_ZONE_GLOBAL_ADDRESS
value: "grpcs://192.168.0.1:5685"
- name: KUMA_MULTIZONE_ZONE_NAME
value: "zone-1"
- name: KUMA_STORE_POSTGRES_PORT
value: "5432"
- name: KUMA_STORE_TYPE
value: "postgres"
args:
- migrate
- up
- --log-level=info
- --config-file=/etc/kuma.io/kuma-control-plane/config.yaml
resources:
limits:
memory: 256Mi
requests:
cpu: 500m
memory: 256Mi
volumeMounts:
- name: kuma-control-plane-config
mountPath: /etc/kuma.io/kuma-control-plane
readOnly: true
containers:
- name: control-plane
image: "docker.io/kumahq/kuma-cp:0.0.1"
imagePullPolicy: IfNotPresent
securityContext:
readOnlyRootFilesystem: true
env:
- name: KUMA_DEFAULTS_SKIP_MESH_CREATION
value: "false"
- name: KUMA_ENVIRONMENT
value: "universal"
- name: KUMA_GENERAL_TLS_CERT_FILE
value: "/var/run/secrets/kuma.io/tls-cert/tls.crt"
- name: KUMA_GENERAL_TLS_KEY_FILE
value: "/var/run/secrets/kuma.io/tls-cert/tls.key"
- name: KUMA_GENERAL_WORK_DIR
value: "/tmp/kuma"
- name: KUMA_MODE
value: "zone"
- name: KUMA_MULTIZONE_ZONE_GLOBAL_ADDRESS
value: "grpcs://192.168.0.1:5685"
- name: KUMA_MULTIZONE_ZONE_NAME
value: "zone-1"
- name: KUMA_STORE_POSTGRES_PORT
value: "5432"
- name: KUMA_STORE_TYPE
value: "postgres"
- name: KUMA_INTER_CP_CATALOG_INSTANCE_ADDRESS
valueFrom:
fieldRef:
fieldPath: status.podIP
args:
- run
- --log-level=info
- --log-output-path=
- --config-file=/etc/kuma.io/kuma-control-plane/config.yaml
ports:
- containerPort: 5680
name: diagnostics
protocol: TCP
- containerPort: 5681
- containerPort: 5682
- containerPort: 5443
- containerPort: 5678
livenessProbe:
timeoutSeconds: 10
httpGet:
path: /healthy
port: 5680
readinessProbe:
timeoutSeconds: 10
httpGet:
path: /ready
port: 5680
resources:
limits:
memory: 256Mi
requests:
cpu: 500m
memory: 256Mi

volumeMounts:
- name: general-tls-cert
mountPath: /var/run/secrets/kuma.io/tls-cert/tls.crt
subPath: tls.crt
readOnly: true
- name: general-tls-cert
mountPath: /var/run/secrets/kuma.io/tls-cert/tls.key
subPath: tls.key
readOnly: true
- name: general-tls-cert
mountPath: /var/run/secrets/kuma.io/tls-cert/ca.crt
subPath: ca.crt
readOnly: true
- name: kuma-control-plane-config
mountPath: /etc/kuma.io/kuma-control-plane
readOnly: true
- name: tmp
mountPath: /tmp
volumes:
- name: general-tls-cert
secret:
secretName: general-tls-secret
- name: kuma-control-plane-config
configMap:
name: kuma-control-plane-config
- name: tmp
emptyDir: {}
73 changes: 36 additions & 37 deletions deployments/charts/kuma/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -176,42 +176,6 @@ returns: formatted image string
{{- end -}}

{{- define "kuma.defaultEnv" -}}
{{ if (and (eq .Values.controlPlane.environment "universal") (not (eq .Values.controlPlane.mode "global"))) }}
{{ fail "Currently you can only run universal mode on kubernetes in a global mode, this limitation might be lifted in the future" }}
{{ end }}
{{ if not (or (eq .Values.controlPlane.mode "zone") (eq .Values.controlPlane.mode "global") (eq .Values.controlPlane.mode "standalone")) }}
{{ $msg := printf "controlPlane.mode invalid got:'%s' supported values: global,zone,standalone" .Values.controlPlane.mode }}
{{ fail $msg }}
{{ end }}
{{ if eq .Values.controlPlane.mode "zone" }}
{{ if empty .Values.controlPlane.zone }}
{{ fail "Can't have controlPlane.zone to be empty when controlPlane.mode=='zone'" }}
{{ else }}
{{ if gt (len .Values.controlPlane.zone) 253 }}
{{ fail "controlPlane.zone must be no more than 253 characters" }}
{{ else }}
{{ if not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" .Values.controlPlane.zone) }}
{{ fail "controlPlane.zone must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character" }}
{{ end }}
{{ end }}
{{ end }}
{{ if empty .Values.controlPlane.kdsGlobalAddress }}
{{ fail "controlPlane.kdsGlobalAddress can't be empty when controlPlane.mode=='zone', needs to be the global control-plane address" }}
{{ else }}
{{ $url := urlParse .Values.controlPlane.kdsGlobalAddress }}
{{ if not (or (eq $url.scheme "grpcs") (eq $url.scheme "grpc")) }}
{{ $msg := printf "controlPlane.kdsGlobalAddress must be a url with scheme grpcs:// or grpc:// got:'%s'" .Values.controlPlane.kdsGlobalAddress }}
{{ fail $msg }}
{{ end }}
{{ end }}
{{ else }}
{{ if not (empty .Values.controlPlane.zone) }}
{{ fail "Can't specify a controlPlane.zone when controlPlane.mode!='zone'" }}
{{ end }}
{{ if not (empty .Values.controlPlane.kdsGlobalAddress) }}
{{ fail "Can't specify a controlPlane.kdsGlobalAddress when controlPlane.mode!='zone'" }}
{{ end }}
{{ end }}
env:
{{ include "kuma.parentEnv" . }}
- name: KUMA_ENVIRONMENT
Expand Down Expand Up @@ -321,6 +285,15 @@ env:
{{- end }}

{{- define "kuma.universal.defaultEnv" -}}
{{ if eq .Values.controlPlane.mode "zone" }}
{{ if .Values.ingress.enabled }}
{{ fail "Can't have ingress.enabled when running controlPlane.mode=='universal'" }}
{{ end }}
{{ if .Values.egress.enabled }}
{{ fail "Can't have egress.enabled when running controlPlane.mode=='universal'" }}
{{ end }}
{{ end }}

env:
- name: KUMA_GENERAL_WORK_DIR
value: "/tmp/kuma"
Expand All @@ -332,8 +305,34 @@ env:
value: "{{ .Values.postgres.port }}"
- name: KUMA_DEFAULTS_SKIP_MESH_CREATION
value: {{ .Values.controlPlane.defaults.skipMeshCreation | quote }}
{{ if and (eq .Values.controlPlane.mode "zone") .Values.controlPlane.tls.general.secretName }}
- name: KUMA_GENERAL_TLS_CERT_FILE
value: /var/run/secrets/kuma.io/tls-cert/tls.crt
- name: KUMA_GENERAL_TLS_KEY_FILE
value: /var/run/secrets/kuma.io/tls-cert/tls.key
{{ end }}
- name: KUMA_MODE
value: "global"
value: {{ .Values.controlPlane.mode | quote }}
{{- if eq .Values.controlPlane.mode "zone" }}
- name: KUMA_MULTIZONE_ZONE_GLOBAL_ADDRESS
value: {{ .Values.controlPlane.kdsGlobalAddress }}
{{- end }}
{{- if .Values.controlPlane.zone }}
- name: KUMA_MULTIZONE_ZONE_NAME
value: {{ .Values.controlPlane.zone | quote }}
{{- end }}
{{- if and (eq .Values.controlPlane.mode "zone") (or .Values.controlPlane.tls.kdsZoneClient.secretName .Values.controlPlane.tls.kdsZoneClient.create) }}
- name: KUMA_MULTIZONE_ZONE_KDS_ROOT_CA_FILE
value: /var/run/secrets/kuma.io/kds-client-tls-cert/ca.crt
{{- end }}
{{- if .Values.experimental.deltaKds }}
- name: KUMA_EXPERIMENTAL_KDS_DELTA_ENABLED
value: "true"
{{- end }}
{{- if .Values.controlPlane.tls.kdsZoneClient.skipVerify }}
- name: KUMA_MULTIZONE_ZONE_KDS_TLS_SKIP_VERIFY
value: "true"
{{- end }}
{{- if .Values.controlPlane.tls.apiServer.secretName }}
- name: KUMA_API_SERVER_HTTPS_TLS_CERT_FILE
value: /var/run/secrets/kuma.io/api-server-tls-cert/tls.crt
Expand Down
Loading

0 comments on commit 671b992

Please sign in to comment.