From 830f9c37bca807c3bb16b0ca6a6f11d6d97cb616 Mon Sep 17 00:00:00 2001 From: weekface Date: Mon, 16 Mar 2020 11:46:02 +0800 Subject: [PATCH 1/4] let tidb-initializer support TLS --- .../tidb_initializer_controller.go | 2 + pkg/manager/member/template.go | 8 ++++ pkg/manager/member/tidb_init_manager.go | 47 +++++++++++++++++-- pkg/util/util.go | 5 ++ 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pkg/controller/tidbinitializer/tidb_initializer_controller.go b/pkg/controller/tidbinitializer/tidb_initializer_controller.go index b7ed97b847..58becb489d 100644 --- a/pkg/controller/tidbinitializer/tidb_initializer_controller.go +++ b/pkg/controller/tidbinitializer/tidb_initializer_controller.go @@ -64,6 +64,7 @@ func NewController( recorder := eventBroadcaster.NewRecorder(v1alpha1.Scheme, corev1.EventSource{Component: "tidbinitializer"}) tidbInitializerInformer := informerFactory.Pingcap().V1alpha1().TidbInitializers() + tidbClusterInformer := informerFactory.Pingcap().V1alpha1().TidbClusters() jobInformer := kubeInformerFactory.Batch().V1().Jobs() typedControl := controller.NewTypedControl(controller.NewRealGenericControl(genericCli, recorder)) @@ -75,6 +76,7 @@ func NewController( jobInformer.Lister(), genericCli, tidbInitializerInformer.Lister(), + tidbClusterInformer.Lister(), typedControl, ), ), diff --git a/pkg/manager/member/template.go b/pkg/manager/member/template.go index cd0330dc78..5cd56c86c7 100644 --- a/pkg/manager/member/template.go +++ b/pkg/manager/member/template.go @@ -281,7 +281,11 @@ var tidbInitStartScriptTpl = template.Must(template.New("tidb-init-start-script" host = '{{ .ClusterName }}-tidb' permit_host = '{{ .PermitHost }}' port = 4000 +{{- if .TLS }} +conn = MySQLdb.connect(host=host, port=port, user='root', connect_timeout=5, ssl={'ca': '{{ .CAPath }}', 'cert': '{{ .CertPath }}', 'key': '{{ .KeyPath }}'}) +{{- else }} conn = MySQLdb.connect(host=host, port=port, user='root', connect_timeout=5) +{{- end }} {{- if .PasswordSet }} password_dir = '/etc/tidb/password' for file in os.listdir(password_dir): @@ -313,6 +317,10 @@ type TiDBInitStartScriptModel struct { PermitHost string PasswordSet bool InitSQL bool + TLS bool + CAPath string + CertPath string + KeyPath string } func RenderTiDBInitStartScript(model *TiDBInitStartScriptModel) (string, error) { diff --git a/pkg/manager/member/tidb_init_manager.go b/pkg/manager/member/tidb_init_manager.go index b638782578..c48aa85d72 100644 --- a/pkg/manager/member/tidb_init_manager.go +++ b/pkg/manager/member/tidb_init_manager.go @@ -15,6 +15,7 @@ package member import ( "fmt" + "github.com/pingcap/tidb-operator/pkg/util" "path" batchv1 "k8s.io/api/batch/v1" @@ -59,6 +60,7 @@ type tidbInitManager struct { jobLister batchlisters.JobLister genericCli client.Client tiLister listers.TidbInitializerLister + tcLister listers.TidbClusterLister typedControl controller.TypedControlInterface } @@ -67,12 +69,14 @@ func NewTiDBInitManager( jobLister batchlisters.JobLister, genericCli client.Client, tiLister listers.TidbInitializerLister, + tcLister listers.TidbClusterLister, typedControl controller.TypedControlInterface, ) InitManager { return &tidbInitManager{ jobLister, genericCli, tiLister, + tcLister, typedControl, } } @@ -134,6 +138,7 @@ func (tm *tidbInitManager) syncTiDBInitConfigMap(ti *v1alpha1.TidbInitializer) e name := controller.TiDBInitializerMemberName(ti.Spec.Clusters.Name) ns := ti.Namespace cm := &corev1.ConfigMap{} + tcName := ti.Spec.Clusters.Name exist, err := tm.typedControl.Exist(client.ObjectKey{ Namespace: ns, @@ -146,7 +151,12 @@ func (tm *tidbInitManager) syncTiDBInitConfigMap(ti *v1alpha1.TidbInitializer) e return nil } - newCm, err := getTiDBInitConfigMap(ti) + tc, err := tm.tcLister.TidbClusters(ns).Get(tcName) + if err != nil { + return err + } + + newCm, err := getTiDBInitConfigMap(ti, tc.Spec.TiDB.IsTLSClientEnabled()) if err != nil { return err } @@ -187,6 +197,13 @@ func (tm *tidbInitManager) syncTiDBInitJob(ti *v1alpha1.TidbInitializer) error { func (tm *tidbInitManager) makeTiDBInitJob(ti *v1alpha1.TidbInitializer) (*batchv1.Job, error) { jobName := controller.TiDBInitializerMemberName(ti.Spec.Clusters.Name) + ns := ti.Namespace + tcName := ti.Spec.Clusters.Name + + tc, err := tm.tcLister.TidbClusters(ns).Get(tcName) + if err != nil { + return nil, err + } var envs []corev1.EnvVar if ti.Spec.Timezone != "" { @@ -207,6 +224,21 @@ func (tm *tidbInitManager) makeTiDBInitJob(ti *v1alpha1.TidbInitializer) (*batch var vms []corev1.VolumeMount var vs []corev1.Volume + if tc.Spec.TiDB.IsTLSClientEnabled() { + vms = append(vms, corev1.VolumeMount{ + Name: "tidb-client-tls", + ReadOnly: true, + MountPath: util.TiDBClientTLSPath, + }) + vs = append(vs, corev1.Volume{ + Name: "tidb-client-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: util.TiDBClientTLSSecretName(tcName), + }, + }, + }) + } vms = append(vms, corev1.VolumeMount{ Name: startKey, ReadOnly: true, @@ -335,7 +367,7 @@ func (tm *tidbInitManager) makeTiDBInitJob(ti *v1alpha1.TidbInitializer) (*batch return job, nil } -func getTiDBInitConfigMap(ti *v1alpha1.TidbInitializer) (*corev1.ConfigMap, error) { +func getTiDBInitConfigMap(ti *v1alpha1.TidbInitializer, tlsClientEnabled bool) (*corev1.ConfigMap, error) { var initSQL, passwdSet bool permitHost := ti.GetPermitHost() @@ -354,12 +386,19 @@ func getTiDBInitConfigMap(ti *v1alpha1.TidbInitializer) (*corev1.ConfigMap, erro return nil, err } - startScript, err := RenderTiDBInitStartScript(&TiDBInitStartScriptModel{ + initModel := &TiDBInitStartScriptModel{ ClusterName: ti.Spec.Clusters.Name, PermitHost: permitHost, InitSQL: initSQL, PasswordSet: passwdSet, - }) + } + if tlsClientEnabled { + initModel.TLS = true + initModel.CAPath = path.Join(util.TiDBClientTLSPath, corev1.ServiceAccountRootCAKey) + initModel.CertPath = path.Join(util.TiDBClientTLSPath, corev1.TLSCertKey) + initModel.KeyPath = path.Join(util.TiDBClientTLSPath, corev1.TLSPrivateKeyKey) + } + startScript, err := RenderTiDBInitStartScript(initModel) if err != nil { return nil, err } diff --git a/pkg/util/util.go b/pkg/util/util.go index f1b571f4ac..bc02d7ad8d 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -30,6 +30,7 @@ import ( var ( ClusterClientTLSPath = "/var/lib/cluster-client-tls" + TiDBClientTLSPath = "/var/lib/tidb-client-tls" ) func GetOrdinalFromPodName(podName string) (int32, error) { @@ -175,3 +176,7 @@ func ClusterClientTLSSecretName(tcName string) string { func ClusterTLSSecretName(tcName, component string) string { return fmt.Sprintf("%s-%s-cluster-secret", tcName, component) } + +func TiDBClientTLSSecretName(tcName string) string { + return fmt.Sprintf("%s-tidb-client-secret", tcName) +} From 05e82bb8c2693a0fa6d05d25737512108a64efea Mon Sep 17 00:00:00 2001 From: weekface Date: Mon, 16 Mar 2020 15:09:30 +0800 Subject: [PATCH 2/4] address comment --- pkg/manager/member/tidb_init_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/manager/member/tidb_init_manager.go b/pkg/manager/member/tidb_init_manager.go index c48aa85d72..065995caa9 100644 --- a/pkg/manager/member/tidb_init_manager.go +++ b/pkg/manager/member/tidb_init_manager.go @@ -15,7 +15,6 @@ package member import ( "fmt" - "github.com/pingcap/tidb-operator/pkg/util" "path" batchv1 "k8s.io/api/batch/v1" @@ -31,6 +30,7 @@ import ( listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/util" ) const ( From d07f9cd9f2acd76b3f29a67243caae1e0240e6d4 Mon Sep 17 00:00:00 2001 From: weekface Date: Mon, 16 Mar 2020 17:07:21 +0800 Subject: [PATCH 3/4] use should create client certificate when tlsClient set true --- charts/tidb-cluster/values.yaml | 5 ++++- pkg/apis/pingcap/v1alpha1/types.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index ebe10a85b4..46e730f0bf 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -465,7 +465,10 @@ tidb: # 2. Create a K8s Secret object which contains the TiDB server-side certificate created above. # The name of this Secret must be: -tidb-server-secret. # kubectl create secret generic -tidb-server-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= - # 3. Then create the TiDB cluster with `tlsClient.enabled` set to `true`. + # 3. Create a K8s Secret object which contains the TiDB client-side certificate created above which will be used by TiDB Operator. + # The name of this Secret must be: -tidb-client-secret. + # kubectl create secret generic -tidb-client-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + # 4. Then create the TiDB cluster with `tlsClient.enabled` set to `true`. enabled: false # mysqlClient is used to set password for TiDB diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 82363dba05..99997e4f8c 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -613,7 +613,10 @@ type TiDBTLSClient struct { // 2. Create a K8s Secret object which contains the TiDB server-side certificate created above. // The name of this Secret must be: -tidb-server-secret. // kubectl create secret generic -tidb-server-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= - // 3. Set Enabled to `true`. + // 3. Create a K8s Secret object which contains the TiDB client-side certificate created above which will be used by TiDB Operator. + // The name of this Secret must be: -tidb-client-secret. + // kubectl create secret generic -tidb-client-secret --namespace= --from-file=tls.crt= --from-file=tls.key= --from-file=ca.crt= + // 4. Set Enabled to `true`. // +optional Enabled bool `json:"enabled,omitempty"` } From d13ef433a88475999954056508657b76b5f141fa Mon Sep 17 00:00:00 2001 From: weekface Date: Tue, 17 Mar 2020 15:07:48 +0800 Subject: [PATCH 4/4] fix ci --- docs/api-references/docs.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/api-references/docs.html b/docs/api-references/docs.html index 11f503b00f..108777a137 100644 --- a/docs/api-references/docs.html +++ b/docs/api-references/docs.html @@ -7665,7 +7665,10 @@

TiDBTLSClient 2. Create a K8s Secret object which contains the TiDB server-side certificate created above. The name of this Secret must be: -tidb-server-secret. kubectl create secret generic -tidb-server-secret –namespace= –from-file=tls.crt= –from-file=tls.key= –from-file=ca.crt= -3. Set Enabled to true.

+3. Create a K8s Secret object which contains the TiDB client-side certificate created above which will be used by TiDB Operator. +The name of this Secret must be: -tidb-client-secret. +kubectl create secret generic -tidb-client-secret –namespace= –from-file=tls.crt= –from-file=tls.key= –from-file=ca.crt= +4. Set Enabled to true.