Automated cherry pick of #1870: TLS between TiDB components (#1916)

charts/tidb-cluster/templates/_helpers.tpl

@@ -28,7 +28,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- end -}}
 
 {{- define "cluster.scheme" -}}
-{{ if .Values.enableTLSCluster }}https{{ else }}http{{ end }}
+{{ if and .Values.tlsCluster .Values.tlsCluster.enabled  }}https{{ else }}http{{ end }}
 {{- end -}}
 
 {{/*
@@ -41,9 +41,9 @@ config-file: |-
     {{- if .Values.pd.config }}
 {{ .Values.pd.config | indent 2 }}
     {{- end -}}
-    {{- if .Values.enableTLSCluster }}
+    {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
   [security]
-  cacert-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  cacert-path = "/var/lib/pd-tls/ca.crt"
   cert-path = "/var/lib/pd-tls/tls.crt"
   key-path = "/var/lib/pd-tls/tls.key"
     {{- end -}}
@@ -64,9 +64,9 @@ config-file: |-
     {{- if .Values.tikv.config }}
 {{ .Values.tikv.config | indent 2 }}
     {{- end -}}
-    {{- if .Values.enableTLSCluster }}
+    {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
   [security]
-  ca-path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  ca-path = "/var/lib/tikv-tls/ca.crt"
   cert-path = "/var/lib/tikv-tls/tls.crt"
   key-path = "/var/lib/tikv-tls/tls.key"
     {{- end -}}
@@ -91,11 +91,11 @@ config-file: |-
     {{- if .Values.tidb.config }}
 {{ .Values.tidb.config | indent 2 }}
     {{- end -}}
-    {{- if or .Values.enableTLSCluster (and .Values.tidb.tlsClient .Values.tidb.tlsClient.enabled) }}
+    {{- if or (and .Values.tlsCluster .Values.tlsCluster.enabled) (and .Values.tidb.tlsClient .Values.tidb.tlsClient.enabled) }}
   [security]
     {{- end -}}
-    {{- if .Values.enableTLSCluster }}
-  cluster-ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+    {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+  cluster-ssl-ca = "/var/lib/tidb-tls/ca.crt"
   cluster-ssl-cert = "/var/lib/tidb-tls/tls.crt"
   cluster-ssl-key = "/var/lib/tidb-tls/tls.key"
     {{- end -}}
@@ -122,7 +122,7 @@ Encapsulate pump configmap data for consistent digest calculation
 pump-config: |-
     {{- if .Values.binlog.pump.config }}
 {{ .Values.binlog.pump.config | indent 2 }}
-    {{- if .Values.enableTLSCluster }}
+    {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
   [security]
   ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
   ssl-cert = "/var/lib/pump-tls/tls.crt"

charts/tidb-cluster/templates/config/_prometheus-config.tpl

@@ -21,10 +21,10 @@ scrape_configs:
   {{- end }}
   tls_config:
     insecure_skip_verify: true
-  {{- if .Values.enableTLSCluster }}
-    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    cert_file: /var/lib/pd-client-tls/tls.crt
-    key_file: /var/lib/pd-client-tls/tls.key
+  {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+    ca_file: /var/lib/cluster-client-tls/ca.crt
+    cert_file: /var/lib/cluster-client-tls/tls.crt
+    key_file: /var/lib/cluster-client-tls/tls.key
   scheme: https
   {{- end }}
   relabel_configs:
@@ -73,10 +73,10 @@ scrape_configs:
   {{- end }}
   tls_config:
     insecure_skip_verify: true
-  {{- if .Values.enableTLSCluster }}
-    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-    cert_file: /var/lib/pd-client-tls/tls.crt
-    key_file: /var/lib/pd-client-tls/tls.key
+  {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+    ca_file: /var/lib/cluster-client-tls/ca.crt
+    cert_file: /var/lib/cluster-client-tls/tls.crt
+    key_file: /var/lib/cluster-client-tls/tls.key
   scheme: https
   {{- end }}
   relabel_configs:
@@ -127,10 +127,10 @@ scrape_configs:
     insecure_skip_verify: true
 # TiKV doesn't support scheme https for now. 
 # And we should fix it after TiKV fix this issue: https://github.com/tikv/tikv/issues/5340
-# {{- if .Values.enableTLSCluster }}
-#     ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-#     cert_file: /var/lib/pd-client-tls/tls.crt
-#     key_file: /var/lib/pd-client-tls/tls.key
+# {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+#     ca_file: /var/lib/cluster-client-tls/ca.crt
+#     cert_file: /var/lib/cluster-client-tls/tls.crt
+#     key_file: /var/lib/cluster-client-tls/tls.key
 #   scheme: https
 # {{- end }}
   relabel_configs:

charts/tidb-cluster/templates/config/_pump-config.tpl

@@ -35,7 +35,7 @@ sync-log = {{ .Values.binlog.pump.syncLog | default true }}
 # write-buffer = 67108864
 # write-L0-pause-trigger = 24
 # write-L0-slowdown-trigger = 17
-{{ if .Values.enableTLSCluster }}
+{{ if and .Values.tlsCluster .Values.tlsCluster.enabled }}
 [security]
 # Path of file that contains list of trusted SSL CAs for connection with cluster components.
 ssl-ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"

charts/tidb-cluster/templates/monitor-deployment.yaml

@@ -134,9 +134,9 @@ spec:
           - name: prometheus-rules
             mountPath: /prometheus-rules
             readOnly: false
-          {{- if .Values.enableTLSCluster }}
-          - name: tls-pd-client
-            mountPath: /var/lib/pd-client-tls
+          {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+          - name: cluster-client-tls
+            mountPath: /var/lib/cluster-client-tls
             readOnly: true
           {{- end }}
       {{- if .Values.monitor.grafana.create }}
@@ -241,11 +241,11 @@ spec:
         name: prometheus-rules
       - emptyDir: {}
         name: grafana-dashboard
-      {{- if .Values.enableTLSCluster }}
-      - name: tls-pd-client
+      {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
+      - name: cluster-client-tls
         secret:
           defaultMode: 420
-          secretName: {{ .Release.Name }}-pd-client
+          secretName: {{ .Release.Name }}-cluster-client-secret
       {{- end }}
     {{- if .Values.monitor.tolerations }}
       tolerations:

charts/tidb-cluster/templates/pump-statefulset.yaml

@@ -55,7 +55,7 @@ spec:
           mountPath: /data
         - name: config
           mountPath: /etc/pump
-      {{- if .Values.enableTLSCluster }}
+      {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
         - name: pump-tls
           mountPath: /var/lib/pump-tls
           readOnly: true
@@ -78,7 +78,7 @@ spec:
           items:
           - key: pump-config
             path: pump.toml
-    {{- if .Values.enableTLSCluster }}
+    {{- if and .Values.tlsCluster .Values.tlsCluster.enabled }}
       - name: pump-tls
         secret:
           secretName: {{ include "pump.tlsSecretName" . }}

charts/tidb-cluster/templates/tidb-cluster.yaml

@@ -21,7 +21,10 @@ spec:
   pvReclaimPolicy: {{ .Values.pvReclaimPolicy }}
   enablePVReclaim: {{ .Values.enablePVReclaim }}
   timezone: {{ .Values.timezone | default "UTC" }}
-  enableTLSCluster: {{ .Values.enableTLSCluster | default false }}
+{{- if .Values.tlsCluster }}
+  tlsCluster:
+{{ toYaml .Values.tlsCluster | indent 4 }}
+{{- end }}
   services:
 {{ toYaml .Values.services | indent 4 }}
   schedulerName: {{ .Values.schedulerName | default "default-scheduler" }}

charts/tidb-cluster/values.yaml

@@ -55,10 +55,23 @@ discovery:
 # if the ConfigMap was not changed.
 enableConfigMapRollout: true
 
-# Whether enable TLS connections between server nodes.
-# When enabled, PD/TiDB/TiKV/PUMP will use TLS encrypted connections to transfer data between each node,
-# certificates will be generated automatically (if not already present).
-enableTLSCluster: false
+# Whether enable the TLS connection between TiDB server components
+tlsCluster:
+  # The steps to enable this feature:
+  #   1. Generate TiDB server components certificates and a client-side certifiacete for them.
+  #      There are multiple ways to generate these certificates:
+  #        - user-provided certificates: https://pingcap.com/docs/stable/how-to/secure/generate-self-signed-certificates/
+  #        - use the K8s built-in certificate signing system signed certificates: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/
+  #        - or use cert-manager signed certificates: https://cert-manager.io/
+  #   2. Create one secret object for one component which contains the certificates created above.
+  #      The name of this Secret must be: <clusterName>-<componentName>-cluster-secret.
+  #        For PD: kubectl create secret generic <clusterName>-pd-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+  #        For TiKV: kubectl create secret generic <clusterName>-tikv-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+  #        For TiDB: kubectl create secret generic <clusterName>-tidb-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+  #        For Client: kubectl create secret generic <clusterName>-cluster-client-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+  #        Same for other components.
+  #   3. Then create the TiDB cluster with `tlsCluster.enabled` set to `true`.
+  enabled: false
 
 pd:
   # Please refer to https://github.com/pingcap/pd/blob/master/conf/config.toml for the default
@@ -618,7 +631,7 @@ binlog:
     # pump configurations (change to the tags of your pump version),
     # just follow the format in the file and configure in the 'config' section
     # as below if you want to customize any configuration.
-    # [security] section will be generated automatically if enableTLSCluster is set to true so users do not need to configure it.
+    # [security] section will be generated automatically if tlsCluster.enabled is set to true so users do not need to configure it.
     # config: |
     #   gc = 7
     #   heartbeat-interval = 2

manifests/crd.yaml

@@ -675,10 +675,6 @@ spec:
               description: 'Whether enable PVC reclaim for orphan PVC left by statefulset
                 scale-in Optional: Defaults to false'
               type: boolean
-            enableTLSCluster:
-              description: 'Enable TLS connection between TiDB server components Optional:
-                Defaults to false'
-              type: boolean
             helper:
               description: HelperSpec contains details of helper component
               properties:
@@ -6055,6 +6051,7 @@ spec:
             timezone:
               description: 'Time zone of TiDB cluster Pods Optional: Defaults to UTC'
               type: string
+            tlsCluster: {}
             tolerations:
               description: Base tolerations of TiDB cluster Pods, components may add
                 more tolerations upon this respectively

pkg/apis/pingcap/v1alpha1/defaulting/tidbcluster.go

@@ -40,9 +40,8 @@ func setTidbClusterSpecDefault(tc *v1alpha1.TidbCluster) {
 	if string(tc.Spec.ImagePullPolicy) == "" {
 		tc.Spec.ImagePullPolicy = corev1.PullIfNotPresent
 	}
-	if tc.Spec.EnableTLSCluster == nil {
-		d := false
-		tc.Spec.EnableTLSCluster = &d
+	if tc.Spec.TLSCluster == nil {
+		tc.Spec.TLSCluster = &v1alpha1.TLSCluster{Enabled: false}
 	}
 	if tc.Spec.EnablePVReclaim == nil {
 		d := false

pkg/apis/pingcap/v1alpha1/tidbcluster.go

@@ -300,11 +300,7 @@ func (tc *TidbCluster) GetClusterID() string {
 }
 
 func (tc *TidbCluster) IsTLSClusterEnabled() bool {
-	enableTLCluster := tc.Spec.EnableTLSCluster
-	if enableTLCluster == nil {
-		return defaultEnableTLSCluster
-	}
-	return *enableTLCluster
+	return tc.Spec.TLSCluster != nil && tc.Spec.TLSCluster.Enabled
 }
 
 func (tc *TidbCluster) Scheme() string {

pkg/apis/pingcap/v1alpha1/types.go

@@ -150,10 +150,10 @@ type TidbClusterSpec struct {
 	// +optional
 	EnablePVReclaim *bool `json:"enablePVReclaim,omitempty"`
 
-	// Enable TLS connection between TiDB server components
-	// Optional: Defaults to false
+	// Whether enable the TLS connection between TiDB server components
+	// Optional: Defaults to nil
 	// +optional
-	EnableTLSCluster *bool `json:"enableTLSCluster,omitempty"`
+	TLSCluster *TLSCluster `json:"tlsCluster,omitempty"`
 
 	// Whether Hostnetwork is enabled for TiDB cluster Pods
 	// Optional: Defaults to false
@@ -618,6 +618,29 @@ type TiDBTLSClient struct {
 	Enabled bool `json:"enabled,omitempty"`
 }
 
+// TLSCluster can enable TLS connection between TiDB server components
+// https://pingcap.com/docs/stable/how-to/secure/enable-tls-between-components/
+type TLSCluster struct {
+	// Enable mutual TLS authentication among TiDB components
+	// Once enabled, the mutual authentication applies to all components,
+	// and it does not support applying to only part of the components.
+	// The steps to enable this feature:
+	//   1. Generate TiDB server components certificates and a client-side certifiacete for them.
+	//      There are multiple ways to generate these certificates:
+	//        - user-provided certificates: https://pingcap.com/docs/stable/how-to/secure/generate-self-signed-certificates/
+	//        - use the K8s built-in certificate signing system signed certificates: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/
+	//        - or use cert-manager signed certificates: https://cert-manager.io/
+	//   2. Create one secret object for one component which contains the certificates created above.
+	//      The name of this Secret must be: <clusterName>-<componentName>-cluster-secret.
+	//        For PD: kubectl create secret generic <clusterName>-pd-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+	//        For TiKV: kubectl create secret generic <clusterName>-tikv-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+	//        For TiDB: kubectl create secret generic <clusterName>-tidb-cluster-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+	//        For Client: kubectl create secret generic <clusterName>-cluster-client-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
+	//        Same for other components.
+	// +optional
+	Enabled bool `json:"enabled,omitempty"`
+}
+
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object