Deploy split proxy/auth with helm chart (#18857)

This commit refactors the `teleport-cluster` Helm chart to deploy separately proxy and auth pods.
It allows users to pass raw teleport configuration to the deployed Teleport nodes.
Finally, it removes the `custom` chart mode as the mode was broken by the split. A new `scratch` mode has been introduced.

See [the corresponding RFD](#18274) describing the design.

examples/chart/teleport-cluster/.lint/cert-manager.yaml

@@ -1,5 +1,10 @@
 clusterName: test-cluster
-chartMode: custom
+chartMode: aws
+aws:
+  region: us-west-2
+  backendTable: test-dynamodb-backend-table
+  auditLogTable: test-dynamodb-auditlog-table
+  sessionRecordingBucket: test-s3-session-storage-bucket
 highAvailability:
   replicaCount: 3
   certManager:

examples/chart/teleport-cluster/.lint/cert-secret.yaml

@@ -1,5 +1,10 @@
 clusterName: test-cluster
-chartMode: custom
+chartMode: aws
+aws:
+  region: us-west-2
+  backendTable: test-dynamodb-backend-table
+  auditLogTable: test-dynamodb-auditlog-table
+  sessionRecordingBucket: test-s3-session-storage-bucket
 annotations:
   certSecret:
     kubernetes.io/cert-secret: value

examples/chart/teleport-cluster/.lint/example-minimal-standalone.yaml

@@ -0,0 +1,7 @@
+# This setup is not safe for production because the proxy will self-sign its certificate.
+# Use those values for testing only
+
+# The chart should deploy and work only with a clusterName.
+# This setup can also cause redirection issues if the proxy is contacted with a hostName instead of an IP address
+# as it is not aware of its external hostname and will attempt to perform a redirection.
+clusterName: helm-lint

examples/chart/teleport-cluster/.lint/initcontainers.yaml

@@ -3,3 +3,6 @@ initContainers:
 - name: "teleport-init"
   image: "alpine"
   args: ["echo test"]
+- name: "teleport-init2"
+  image: "alpine"
+  args: ["echo test2"]

examples/chart/teleport-cluster/.lint/pdb.yaml

@@ -1,5 +1,10 @@
 clusterName: helm-lint
-chartMode: custom
+chartMode: aws
+aws:
+  region: us-west-2
+  backendTable: test-dynamodb-backend-table
+  auditLogTable: test-dynamodb-auditlog-table
+  sessionRecordingBucket: test-s3-session-storage-bucket
 highAvailability:
   replicaCount: 3
   podDisruptionBudget:

examples/chart/teleport-cluster/.lint/proxy-listener-mode-separate.yaml

@@ -0,0 +1,2 @@
+clusterName: test-proxy-listener-mode
+proxyListenerMode: separate

examples/chart/teleport-cluster/templates/_helpers.tpl

@@ -2,6 +2,77 @@
 Create the name of the service account to use
 if serviceAccount is not defined or serviceAccount.name is empty, use .Release.Name
 */}}
-{{- define "teleport.serviceAccountName" -}}
+{{- define "teleport-cluster.auth.serviceAccountName" -}}
 {{- coalesce .Values.serviceAccount.name .Release.Name -}}
 {{- end -}}
+
+{{- define "teleport-cluster.proxy.serviceAccountName" -}}
+{{- coalesce .Values.serviceAccount.name .Release.Name -}}-proxy
+{{- end -}}
+
+{{- define "teleport-cluster.version" -}}
+{{- coalesce .Values.teleportVersionOverride .Chart.Version }}
+{{- end -}}
+
+{{- define "teleport-cluster.majorVersion" -}}
+{{- (semver (include "teleport-cluster.version" .)).Major -}}
+{{- end -}}
+
+{{/* Proxy selector labels */}}
+{{- define "teleport-cluster.proxy.selectorLabels" -}}
+app.kubernetes.io/name: '{{ default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}'
+app.kubernetes.io/instance: '{{ .Release.Name }}'
+app.kubernetes.io/component: 'proxy'
+{{- end -}}
+
+{{/* Proxy all labels */}}
+{{- define "teleport-cluster.proxy.labels" -}}
+{{ include "teleport-cluster.proxy.selectorLabels" . }}
+helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}'
+app.kubernetes.io/managed-by: '{{ .Release.Service }}'
+app.kubernetes.io/version: '{{ include "teleport-cluster.version" . }}'
+teleport.dev/majorVersion: '{{ include "teleport-cluster.majorVersion" . }}'
+{{- end -}}
+
+{{/* Auth pods selector labels */}}
+{{- define "teleport-cluster.auth.selectorLabels" -}}
+app.kubernetes.io/name: '{{ default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}'
+app.kubernetes.io/instance: '{{ .Release.Name }}'
+app.kubernetes.io/component: 'auth'
+{{- end -}}
+
+{{/* All pods all labels */}}
+{{- define "teleport-cluster.labels" -}}
+{{ include "teleport-cluster.selectorLabels" . }}
+helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}'
+app.kubernetes.io/managed-by: '{{ .Release.Service }}'
+app.kubernetes.io/version: '{{ include "teleport-cluster.version" . }}'
+teleport.dev/majorVersion: '{{ include "teleport-cluster.majorVersion" . }}'
+{{- end -}}
+
+{{/* All pods selector labels */}}
+{{- define "teleport-cluster.selectorLabels" -}}
+app.kubernetes.io/name: '{{ default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}'
+app.kubernetes.io/instance: '{{ .Release.Name }}'
+{{- end -}}
+
+{{/* Auth pods all labels */}}
+{{- define "teleport-cluster.auth.labels" -}}
+{{ include "teleport-cluster.auth.selectorLabels" . }}
+helm.sh/chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}'
+app.kubernetes.io/managed-by: '{{ .Release.Service }}'
+app.kubernetes.io/version: '{{ include "teleport-cluster.version" . }}'
+teleport.dev/majorVersion: '{{ include "teleport-cluster.majorVersion" . }}'
+{{- end -}}
+
+{{/* ServiceNames are limited to 63 characters, we might have to truncate the ReleaseName
+     to make sure the auth serviceName won't exceed this limit */}}
+{{- define "teleport-cluster.auth.serviceName" -}}
+{{- .Release.Name | trunc 58 | trimSuffix "-" -}}-auth
+{{- end -}}
+
+{{/* In most places we want to use the FQDN instead of relying on Kubernetes ndots behaviour
+     for performance reasons */}}
+{{- define "teleport-cluster.auth.serviceFQDN" -}}
+{{ include "teleport-cluster.auth.serviceName" . }}.{{ .Release.Namespace }}.svc.cluster.local
+{{- end -}}

examples/chart/teleport-cluster/templates/auth/_config.aws.tpl

@@ -0,0 +1,25 @@
+{{- define "teleport-cluster.auth.config.aws" -}}
+{{ include "teleport-cluster.auth.config.common" . }}
+  storage:
+    type: dynamodb
+    region: {{ required "aws.region is required in chart values" .Values.aws.region }}
+    table_name: {{ required "aws.backendTable is required in chart values" .Values.aws.backendTable }}
+    {{- if .Values.aws.auditLogMirrorOnStdout }}
+    audit_events_uri: ['dynamodb://{{ required "aws.auditLogTable is required in chart values" .Values.aws.auditLogTable }}', 'stdout://']
+    {{- else }}
+    audit_events_uri: ['dynamodb://{{ required "aws.auditLogTable is required in chart values" .Values.aws.auditLogTable }}']
+    {{- end }}
+    audit_sessions_uri: s3://{{ required "aws.sessionRecordingBucket is required in chart values" .Values.aws.sessionRecordingBucket }}
+    continuous_backups: {{ required "aws.backups is required in chart values" .Values.aws.backups }}
+    {{- if .Values.aws.dynamoAutoScaling }}
+    auto_scaling: true
+    read_min_capacity: {{ required "aws.readMinCapacity is required when aws.dynamoAutoScaling is true" .Values.aws.readMinCapacity }}
+    read_max_capacity: {{ required "aws.readMaxCapacity is required when aws.dynamoAutoScaling is true" .Values.aws.readMaxCapacity }}
+    read_target_value: {{ required "aws.readTargetValue is required when aws.dynamoAutoScaling is true" .Values.aws.readTargetValue }}
+    write_min_capacity: {{ required "aws.writeMinCapacity is required when aws.dynamoAutoScaling is true" .Values.aws.writeMinCapacity }}
+    write_max_capacity: {{ required "aws.writeMaxCapacity is required when aws.dynamoAutoScaling is true" .Values.aws.writeMaxCapacity }}
+    write_target_value: {{ required "aws.writeTargetValue is required when aws.dynamoAutoScaling is true" .Values.aws.writeTargetValue }}
+    {{- else }}
+    auto_scaling: false
+    {{- end }}
+{{- end -}}

examples/chart/teleport-cluster/templates/auth/_config.common.tpl

@@ -0,0 +1,64 @@
+{{- define "teleport-cluster.auth.config.common" -}}
+{{- $authentication := mustMergeOverwrite .Values.authentication (default dict .Values.authenticationSecondFactor) -}}
+{{- $logLevel := (coalesce .Values.logLevel .Values.log.level "INFO") -}}
+version: v3
+kubernetes_service:
+  enabled: true
+  listen_addr: 0.0.0.0:3026
+  public_addr: "{{ include "teleport-cluster.auth.serviceFQDN" . }}:3026"
+{{- if .Values.kubeClusterName }}
+  kube_cluster_name: {{ .Values.kubeClusterName }}
+{{- else }}
+  kube_cluster_name: {{ .Values.clusterName }}
+{{- end }}
+{{- if .Values.labels }}
+  labels: {{- toYaml .Values.labels | nindent 8 }}
+{{- end }}
+proxy_service:
+  enabled: false
+ssh_service:
+  enabled: false
+auth_service:
+  enabled: true
+  cluster_name: {{ required "clusterName is required in chart values" .Values.clusterName }}
+{{- if .Values.enterprise }}
+  license_file: '/var/lib/license/license.pem'
+{{- end }}
+  authentication:
+    type: "{{ required "authentication.type is required in chart values" (coalesce .Values.authenticationType $authentication.type) }}"
+    local_auth: {{ $authentication.localAuth }}
+{{- if $authentication.connectorName }}
+    connector_name: "{{ $authentication.connectorName }}"
+{{- end }}
+{{- if $authentication.lockingMode }}
+    locking_mode: "{{ $authentication.lockingMode }}"
+{{- end }}
+{{- if $authentication.secondFactor }}
+    second_factor: "{{ $authentication.secondFactor }}"
+  {{- if not (or (eq $authentication.secondFactor "off") (eq $authentication.secondFactor "otp")) }}
+    webauthn:
+      rp_id: {{ required "clusterName is required in chart values" .Values.clusterName }}
+    {{- if $authentication.webauthn }}
+      {{- if $authentication.webauthn.attestationAllowedCas }}
+      attestation_allowed_cas: {{- toYaml $authentication.webauthn.attestationAllowedCas | nindent 12 }}
+      {{- end }}
+      {{- if $authentication.webauthn.attestationDeniedCas }}
+      attestation_denied_cas: {{- toYaml $authentication.webauthn.attestationDeniedCas | nindent 12 }}
+      {{- end }}
+    {{- end }}
+  {{- end }}
+{{- end }}
+{{- if .Values.sessionRecording }}
+  session_recording: {{ .Values.sessionRecording }}
+{{- end }}
+{{- if .Values.proxyListenerMode }}
+  proxy_listener_mode: {{ .Values.proxyListenerMode }}
+{{- end }}
+teleport:
+  log:
+    severity: {{ $logLevel }}
+    output: {{ .Values.log.output }}
+    format:
+      output: {{ .Values.log.format }}
+      extra_fields: {{ .Values.log.extraFields | toJson }}
+{{- end -}}

examples/chart/teleport-cluster/templates/auth/_config.gcp.tpl

@@ -0,0 +1,16 @@
+{{- define "teleport-cluster.auth.config.gcp" -}}
+{{ include "teleport-cluster.auth.config.common" . }}
+  storage:
+    type: firestore
+    project_id: {{ required "gcp.projectId is required in chart values" .Values.gcp.projectId }}
+    collection_name: {{ required "gcp.backendTable is required in chart values" .Values.gcp.backendTable }}
+    {{- if .Values.gcp.credentialSecretName }}
+    credentials_path: /etc/teleport-secrets/gcp-credentials.json
+    {{- end }}
+    {{- if .Values.gcp.auditLogMirrorOnStdout }}
+    audit_events_uri: ['firestore://{{ required "gcp.auditLogTable is required in chart values" .Values.gcp.auditLogTable }}?projectID={{ required "gcp.projectId is required in chart values" .Values.gcp.projectId }}{{ empty .Values.gcp.credentialSecretName | ternary "" "&credentialsPath=/etc/teleport-secrets/gcp-credentials.json"}}', 'stdout://']
+    {{- else }}
+    audit_events_uri: ['firestore://{{ required "gcp.auditLogTable is required in chart values" .Values.gcp.auditLogTable }}?projectID={{ required "gcp.projectId is required in chart values" .Values.gcp.projectId }}{{ empty .Values.gcp.credentialSecretName | ternary "" "&credentialsPath=/etc/teleport-secrets/gcp-credentials.json"}}']
+    {{- end }}
+    audit_sessions_uri: "gs://{{ required "gcp.sessionRecordingBucket is required in chart values" .Values.gcp.sessionRecordingBucket }}?projectID={{ required "gcp.projectId is required in chart values" .Values.gcp.projectId }}{{ empty .Values.gcp.credentialSecretName | ternary "" "&credentialsPath=/etc/teleport-secrets/gcp-credentials.json"}}"
+{{- end -}}

examples/chart/teleport-cluster/templates/auth/_config.scratch.tpl

@@ -0,0 +1,13 @@
+{{- define "teleport-cluster.auth.config.scratch" -}}
+{{- required "'auth.teleportConfig' is required in scratch mode" .Values.auth.teleportConfig }}
+proxy_service:
+  enabled: false
+ssh_service:
+  enabled: false
+auth_service:
+  enabled: true
+{{- end -}}
+
+{{- define "teleport-cluster.auth.config.custom" -}}
+{{ fail "'custom' mode has been removed with chart v12 because of the proxy/auth split breaking change, see https://goteleport.com/docs/deploy-a-cluster/helm-deployments/migration-v12/" }}
+{{- end -}}

examples/chart/teleport-cluster/templates/auth/_config.standalone.tpl

@@ -0,0 +1,3 @@
+{{- define "teleport-cluster.auth.config.standalone" -}}
+{{ include "teleport-cluster.auth.config.common" . }}
+{{- end -}}

examples/chart/teleport-cluster/templates/auth/clusterrolebinding.yaml

@@ -0,0 +1,31 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Release.Name }}
+  labels: {{- include "teleport-cluster.auth.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Name }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "teleport-cluster.auth.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+---
+# This ClusterRoleBinding allows the auth service-account to validate Kubernetes tokens
+# This is required for proxies to join using their Kubernetes tokens
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Release.Name }}-auth
+  labels: {{- include "teleport-cluster.auth.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: {{ include "teleport-cluster.auth.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}

examples/chart/teleport-cluster/templates/auth/config.yaml

@@ -0,0 +1,28 @@
+{{- $auth := mustMergeOverwrite .Values .Values.auth -}}
+{{- $configTemplate := printf "teleport-cluster.auth.config.%s" $auth.chartMode -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-auth
+  namespace: {{ .Release.Namespace }}
+  labels: {{- include "teleport-cluster.auth.labels" . | nindent 4 }}
+{{- if $auth.annotations.config }}
+  annotations: {{- toYaml $auth.annotations.config | nindent 4 }}
+{{- end }}
+data:
+{{- if $auth.createProxyToken }}
+  apply-on-startup.yaml: |2
+    kind: token
+    version: v2
+    metadata:
+      name: {{ .Release.Name }}-proxy
+      expires: "3000-01-01T00:00:00Z"
+    spec:
+      roles: [Proxy]
+      join_method: kubernetes
+      kubernetes:
+        allow:
+          - service_account: "{{ .Release.Namespace }}:{{ include "teleport-cluster.proxy.serviceAccountName" . }}"
+{{- end }}
+  teleport.yaml: |2
+    {{- mustMergeOverwrite (include $configTemplate . | fromYaml) $auth.teleportConfig | toYaml | nindent 4 -}}