tls: TLS between TiDB components

charts/tidb-cluster/templates/_helpers.tpl

@@ -28,7 +28,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- end -}}
 
 {{- define "cluster.scheme" -}}
-{{ if .Values.enableTLSCluster }}https{{ else }}http{{ end }}
+{{ if and .Values.tlsCluster .Values.tlsCluster.enabled  }}https{{ else }}http{{ end }}
 {{- end -}}
 
 {{/*
@@ -41,9 +41,9 @@ config-file: |-
  {{- if .Values.pd.config }}
 {{ .Values.pd.config | indent 2 }}
  {{- end -}}
- {{- if .Values.enableTLSCluster }}
+ {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
  [security]
- cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+ cacert-path = "/var/lib/pd-tls/ca.crt"
  cert-path = "/var/lib/pd-tls/tls.crt"
  key-path = "/var/lib/pd-tls/tls.key"
  {{- end -}}
@@ -64,9 +64,9 @@ config-file: |-
  {{- if .Values.tikv.config }}
 {{ .Values.tikv.config | indent 2 }}
  {{- end -}}
- {{- if .Values.enableTLSCluster }}
+ {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
  [security]
- ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+ ca-path = "/var/lib/tikv-tls/ca.crt"
  cert-path = "/var/lib/tikv-tls/tls.crt"
  key-path = "/var/lib/tikv-tls/tls.key"
  {{- end -}}
@@ -91,11 +91,11 @@ config-file: |-
  {{- if .Values.tidb.config }}
 {{ .Values.tidb.config | indent 2 }}
  {{- end -}}
- {{- if or .Values.enableTLSCluster (and .Values.tidb.tlsClient .Values.tidb.tlsClient.enabled) }}
+ {{- if or (and .Values.tlsCluster .Values.tlsCluster.enabled) (and .Values.tidb.tlsClient .Values.tidb.tlsClient.enabled) }}
  [security]
  {{- end -}}
- {{- if .Values.enableTLSCluster }}
- cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+ {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+ cluster-ssl-ca = "/var/lib/tidb-tls/ca.crt"
  cluster-ssl-cert = "/var/lib/tidb-tls/tls.crt"
  cluster-ssl-key = "/var/lib/tidb-tls/tls.key"
  {{- end -}}
@@ -122,7 +122,7 @@ Encapsulate pump configmap data for consistent digest calculation
 pump-config: |-
  {{- if .Values.binlog.pump.config }}
 {{ .Values.binlog.pump.config | indent 2 }}
- {{- if .Values.enableTLSCluster }}
+ {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
  [security]
  ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
  ssl-cert = "/var/lib/pump-tls/tls.crt"

charts/tidb-cluster/templates/config/_prometheus-config.tpl

@@ -21,10 +21,10 @@ scrape_configs:
  {{- end }}
  tls_config:
  insecure_skip_verify: true
- {{- if .Values.enableTLSCluster }}
- ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
- cert_file: /var/lib/pd-client-tls/tls.crt
- key_file: /var/lib/pd-client-tls/tls.key
+ {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+ ca_file: /var/lib/cluster-client-tls/ca.crt
+ cert_file: /var/lib/cluster-client-tls/tls.crt
+ key_file: /var/lib/cluster-client-tls/tls.key
  scheme: https
  {{- end }}
  relabel_configs:
@@ -73,10 +73,10 @@ scrape_configs:
  {{- end }}
  tls_config:
  insecure_skip_verify: true
- {{- if .Values.enableTLSCluster }}
- ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
- cert_file: /var/lib/pd-client-tls/tls.crt
- key_file: /var/lib/pd-client-tls/tls.key
+ {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+ ca_file: /var/lib/cluster-client-tls/ca.crt
+ cert_file: /var/lib/cluster-client-tls/tls.crt
+ key_file: /var/lib/cluster-client-tls/tls.key
  scheme: https
  {{- end }}
  relabel_configs:
@@ -127,10 +127,10 @@ scrape_configs:
  insecure_skip_verify: true
 # TiKV doesn't support scheme https for now. 
 # And we should fix it after TiKV fix this issue: https://github.com/tikv/tikv/issues/5340
-# {{- if .Values.enableTLSCluster }}
-# ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-# cert_file: /var/lib/pd-client-tls/tls.crt
-# key_file: /var/lib/pd-client-tls/tls.key
+# {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+# ca_file: /var/lib/cluster-client-tls/ca.crt
+# cert_file: /var/lib/cluster-client-tls/tls.crt
+# key_file: /var/lib/cluster-client-tls/tls.key
 # scheme: https
 # {{- end }}
  relabel_configs:

charts/tidb-cluster/templates/config/_pump-config.tpl

@@ -35,7 +35,7 @@ sync-log = {{ .Values.binlog.pump.syncLog | default true }}
 # write-buffer = 67108864
 # write-L0-pause-trigger = 24
 # write-L0-slowdown-trigger = 17
-{{ if .Values.enableTLSCluster }}
+{{ if and .Values.tlsCluster .Values.tlsCluster.enabled }}
 [security]
 # Path of file that contains list of trusted SSL CAs for connection with cluster components.
 ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"

charts/tidb-cluster/templates/monitor-deployment.yaml

@@ -134,9 +134,9 @@ spec:
  - name: prometheus-rules
  mountPath: /prometheus-rules
  readOnly: false
- {{- if .Values.enableTLSCluster }}
- - name: tls-pd-client
- mountPath: /var/lib/pd-client-tls
+ {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+ - name: cluster-client-tls
+ mountPath: /var/lib/cluster-client-tls
  readOnly: true
  {{- end }}
  {{- if .Values.monitor.grafana.create }}
@@ -241,11 +241,11 @@ spec:
  name: prometheus-rules
  - emptyDir: {}
  name: grafana-dashboard
- {{- if .Values.enableTLSCluster }}
- - name: tls-pd-client
+ {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+ - name: cluster-client-tls
  secret:
  defaultMode: 420
- secretName: {{ .Release.Name }}-pd-client
+ secretName: {{ .Release.Name }}-cluster-client-secret
  {{- end }}
  {{- if .Values.monitor.tolerations }}
  tolerations:

charts/tidb-cluster/templates/pump-statefulset.yaml

@@ -55,7 +55,7 @@ spec:
  mountPath: /data
  - name: config
  mountPath: /etc/pump
- {{- if .Values.enableTLSCluster }}
+ {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
  - name: pump-tls
  mountPath: /var/lib/pump-tls
  readOnly: true
@@ -78,7 +78,7 @@ spec:
  items:
  - key: pump-config
  path: pump.toml
- {{- if .Values.enableTLSCluster }}
+ {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
  - name: pump-tls
  secret:
  secretName: {{ include "pump.tlsSecretName" . }}

charts/tidb-cluster/templates/tidb-cluster.yaml

@@ -21,7 +21,10 @@ spec:
  pvReclaimPolicy: {{ .Values.pvReclaimPolicy }}
  enablePVReclaim: {{ .Values.enablePVReclaim }}
  timezone: {{ .Values.timezone | default "UTC" }}
- enableTLSCluster: {{ .Values.enableTLSCluster | default false }}
+{{- if .Values.tlsCluster }}
+ tlsCluster:
+{{ toYaml .Values.tlsCluster | indent 4 }}
+{{- end }}
  services:
 {{ toYaml .Values.services | indent 4 }}
  schedulerName: {{ .Values.schedulerName | default "default-scheduler" }}

charts/tidb-cluster/values.yaml

@@ -55,10 +55,23 @@ discovery:
 # if the ConfigMap was not changed.
 enableConfigMapRollout: true
 
-# Whether enable TLS connections between server nodes.
-# When enabled, PD/TiDB/TiKV/PUMP will use TLS encrypted connections to transfer data between each node,
-# certificates will be generated automatically (if not already present).
-enableTLSCluster: false
+# Whether enable the TLS connection between TiDB server components
+tlsCluster:
+ # The steps to enable this feature:
+ # 1. Generate TiDB server components certificates and a client-side certifiacete for them.
+ # There are multiple ways to generate these certificates:
+ # - user-provided certificates: https://pingcap.com/docs/stable/how-to/secure/generate-self-signed-certificates/
+ # - use the K8s built-in certificate signing system signed certificates: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/
+ # - or use cert-manager signed certificates: https://cert-manager.io/
+ # 2. Create one secret object for one component which contains the certificates created above.
+ # The name of this Secret must be: <clusterName>-<componentName>-cluster-secret.
+ # For PD: kubectl create secret generic <clusterName>-pd-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+ # For TiKV: kubectl create secret generic <clusterName>-tikv-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+ # For TiDB: kubectl create secret generic <clusterName>-tidb-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+ # For Client: kubectl create secret generic <clusterName>-cluster-client-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+ # Same for other components.
+ # 3. Then create the TiDB cluster with `tlsCluster.enabled` set to `true`.
+ enabled: false
 
 pd:
  # Please refer to https://github.com/pingcap/pd/blob/master/conf/config.toml for the default
@@ -616,7 +629,7 @@ binlog:
  # pump configurations (change to the tags of your pump version),
  # just follow the format in the file and configure in the 'config' section
  # as below if you want to customize any configuration.
- # [security] section will be generated automatically if enableTLSCluster is set to true so users do not need to configure it.
+ # [security] section will be generated automatically if tlsCluster.enabled is set to true so users do not need to configure it.
  # config: |
  # gc = 7
  # heartbeat-interval = 2

manifests/crd.yaml

@@ -675,10 +675,6 @@ spec:
  description: 'Whether enable PVC reclaim for orphan PVC left by statefulset
  scale-in Optional: Defaults to false'
  type: boolean
- enableTLSCluster:
- description: 'Enable TLS connection between TiDB server components Optional:
- Defaults to false'
- type: boolean
  helper:
  description: HelperSpec contains details of helper component
  properties:
@@ -6055,6 +6051,7 @@ spec:
  timezone:
  description: 'Time zone of TiDB cluster Pods Optional: Defaults to UTC'
  type: string
+ tlsCluster: {}
  tolerations:
  description: Base tolerations of TiDB cluster Pods, components may add
  more tolerations upon this respectively

pkg/apis/pingcap/v1alpha1/defaulting/tidbcluster.go

@@ -40,9 +40,8 @@ func setTidbClusterSpecDefault(tc *v1alpha1.TidbCluster) {
  if string(tc.Spec.ImagePullPolicy) == "" {
  tc.Spec.ImagePullPolicy = corev1.PullIfNotPresent
  }
- if tc.Spec.EnableTLSCluster == nil {
- d := false
- tc.Spec.EnableTLSCluster = &d
+ if tc.Spec.TLSCluster == nil {
+ tc.Spec.TLSCluster = &v1alpha1.TLSCluster{Enabled: false}
  }
  if tc.Spec.EnablePVReclaim == nil {
  d := false

pkg/apis/pingcap/v1alpha1/tidbcluster.go

@@ -300,11 +300,7 @@ func (tc *TidbCluster) GetClusterID() string {
 }
 
 func (tc *TidbCluster) IsTLSClusterEnabled() bool {
- enableTLCluster := tc.Spec.EnableTLSCluster
- if enableTLCluster == nil {
- return defaultEnableTLSCluster
- }
- return *enableTLCluster
+ return tc.Spec.TLSCluster != nil && tc.Spec.TLSCluster.Enabled
 }
 
 func (tc *TidbCluster) Scheme() string {

pkg/apis/pingcap/v1alpha1/types.go

@@ -150,10 +150,10 @@ type TidbClusterSpec struct {
  // +optional
  EnablePVReclaim *bool `json:"enablePVReclaim,omitempty"`
 
- // Enable TLS connection between TiDB server components
- // Optional: Defaults to false
+ // Whether enable the TLS connection between TiDB server components
+ // Optional: Defaults to nil
  // +optional
- EnableTLSCluster *bool `json:"enableTLSCluster,omitempty"`
+ TLSCluster *TLSCluster `json:"tlsCluster,omitempty"`
 
  // Whether Hostnetwork is enabled for TiDB cluster Pods
  // Optional: Defaults to false
@@ -618,6 +618,29 @@ type TiDBTLSClient struct {
  Enabled bool `json:"enabled,omitempty"`
 }
 
+// TLSCluster can enable TLS connection between TiDB server components
+// https://pingcap.com/docs/stable/how-to/secure/enable-tls-between-components/
+type TLSCluster struct {
+ // Enable mutual TLS authentication among TiDB components
+ // Once enabled, the mutual authentication applies to all components,
+ // and it does not support applying to only part of the components.
+ // The steps to enable this feature:
+ // 1. Generate TiDB server components certificates and a client-side certifiacete for them.
+ // There are multiple ways to generate these certificates:
+ // - user-provided certificates: https://pingcap.com/docs/stable/how-to/secure/generate-self-signed-certificates/
+ // - use the K8s built-in certificate signing system signed certificates: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/
+ // - or use cert-manager signed certificates: https://cert-manager.io/
+ // 2. Create one secret object for one component which contains the certificates created above.
+ // The name of this Secret must be: <clusterName>-<componentName>-cluster-secret.
+ // For PD: kubectl create secret generic <clusterName>-pd-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+ // For TiKV: kubectl create secret generic <clusterName>-tikv-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+ // For TiDB: kubectl create secret generic <clusterName>-tidb-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+ // For Client: kubectl create secret generic <clusterName>-cluster-client-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+ // Same for other components.
+ // +optional
+ Enabled bool `json:"enabled,omitempty"`
+}
+
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object