From 64925a639a708fee1b0aa4ee43052beab04bcaa0 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Mon, 30 Mar 2020 18:42:20 +0800 Subject: [PATCH] Backup: open mysql client TLS in backup (#2003) (#2066) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 尹亮 <30903849+shuijing198799@users.noreply.github.com> --- cmd/backup-manager/app/backup/manager.go | 13 ++++++- cmd/backup-manager/app/export/manager.go | 10 +++++- cmd/backup-manager/app/import/manager.go | 10 +++++- cmd/backup-manager/app/restore/manager.go | 14 +++++++- cmd/backup-manager/app/util/generic.go | 36 +++++++++++++++++-- docs/api-references/docs.html | 31 ++++++++++++++++ manifests/backup/backup-aws-s3-br.yaml | 6 +++- manifests/backup/backup-s3-br.yaml | 6 +++- .../backup/backup-schedule-aws-s3-br.yaml | 6 +++- manifests/backup/backup-schedule-s3-br.yaml | 6 +++- manifests/backup/restore-aws-s3-br.yaml | 6 +++- manifests/backup/restore-s3-br.yaml | 6 +++- manifests/crd.yaml | 3 ++ .../pingcap/v1alpha1/openapi_generated.go | 8 +++++ pkg/apis/pingcap/v1alpha1/types.go | 10 ++++++ .../pingcap/v1alpha1/zz_generated.deepcopy.go | 9 +++-- pkg/backup/backup/backup_cleaner.go | 2 +- pkg/backup/backup/backup_manager.go | 23 ++++++++++-- pkg/backup/restore/restore_manager.go | 24 +++++++++++-- 19 files changed, 210 insertions(+), 19 deletions(-) diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index f4389b1d48..1bd3d2f829 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -80,6 +80,11 @@ func (bm *Manager) ProcessBackup() error { }) } + enableTLSClient := false + if backup.Spec.From.TLSClient != nil && backup.Spec.From.TLSClient.Enabled { + enableTLSClient = true + } + if backup.Spec.BR == nil { return fmt.Errorf("no br config in %s", bm) } @@ -87,8 +92,14 @@ func (bm *Manager) ProcessBackup() error { bm.setOptions(backup) var db *sql.DB + var dsn string err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err = util.OpenDB(bm.GetDSN()) + dsn, err = bm.GetDSN(enableTLSClient) + if err != nil { + klog.Errorf("can't get dsn of tidb cluster %s, err: %s", bm, err) + return false, err + } + db, err = util.OpenDB(dsn) if err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) return false, nil diff --git a/cmd/backup-manager/app/export/manager.go b/cmd/backup-manager/app/export/manager.go index b4a488e6ef..544dd75598 100644 --- a/cmd/backup-manager/app/export/manager.go +++ b/cmd/backup-manager/app/export/manager.go @@ -83,8 +83,16 @@ func (bm *BackupManager) ProcessBackup() error { bm.setOptions(backup) var db *sql.DB + var dsn string err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err = util.OpenDB(bm.GetDSN()) + // TLS is not currently supported + dsn, err = bm.GetDSN(false) + if err != nil { + klog.Errorf("can't get dsn of tidb cluster %s, err: %s", bm, err) + return false, err + } + + db, err = util.OpenDB(dsn) if err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) return false, nil diff --git a/cmd/backup-manager/app/import/manager.go b/cmd/backup-manager/app/import/manager.go index 3d6ac2ce31..624d24e156 100644 --- a/cmd/backup-manager/app/import/manager.go +++ b/cmd/backup-manager/app/import/manager.go @@ -84,8 +84,16 @@ func (rm *RestoreManager) ProcessRestore() error { rm.setOptions(restore) var db *sql.DB + var dsn string err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err = util.OpenDB(rm.GetDSN()) + // TLS is not currently supported + dsn, err = rm.GetDSN(false) + if err != nil { + klog.Errorf("can't get dsn of tidb cluster %s, err: %s", rm, err) + return false, err + } + + db, err = util.OpenDB(dsn) if err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", rm, err) return false, nil diff --git a/cmd/backup-manager/app/restore/manager.go b/cmd/backup-manager/app/restore/manager.go index 56b06b66f5..5003865f95 100644 --- a/cmd/backup-manager/app/restore/manager.go +++ b/cmd/backup-manager/app/restore/manager.go @@ -82,11 +82,23 @@ func (rm *Manager) ProcessRestore() error { return fmt.Errorf("no br config in %s", rm) } + enableTLSClient := false + if restore.Spec.To.TLSClient != nil && restore.Spec.To.TLSClient.Enabled { + enableTLSClient = true + } + rm.setOptions(restore) var db *sql.DB + var dsn string err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err = util.OpenDB(rm.GetDSN()) + dsn, err = rm.GetDSN(enableTLSClient) + if err != nil { + klog.Errorf("can't get dsn of tidb cluster %s, err: %s", rm, err) + return false, err + } + + db, err = util.OpenDB(dsn) if err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", rm, err) return false, nil diff --git a/cmd/backup-manager/app/util/generic.go b/cmd/backup-manager/app/util/generic.go index 04bebc5b3b..97b6e2fe0d 100644 --- a/cmd/backup-manager/app/util/generic.go +++ b/cmd/backup-manager/app/util/generic.go @@ -14,10 +14,18 @@ package util import ( + "crypto/tls" + "crypto/x509" "database/sql" + "errors" "fmt" + "io/ioutil" + "path" + "github.com/go-sql-driver/mysql" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" + "github.com/pingcap/tidb-operator/pkg/util" + corev1 "k8s.io/api/core/v1" ) // GenericOptions contains the generic input arguments to the backup/restore command @@ -35,8 +43,32 @@ func (bo *GenericOptions) String() string { return fmt.Sprintf("%s/%s", bo.Namespace, bo.ResourceName) } -func (bo *GenericOptions) GetDSN() string { - return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, constants.TidbMetaDB) +func (bo *GenericOptions) GetDSN(enabledTLSClient bool) (string, error) { + if !enabledTLSClient { + return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, constants.TidbMetaDB), nil + } + rootCertPool := x509.NewCertPool() + pem, err := ioutil.ReadFile(path.Join(util.TiDBClientTLSPath, corev1.ServiceAccountRootCAKey)) + if err != nil { + return "", err + } + if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { + return "", errors.New("Failed to append PEM") + } + clientCert := make([]tls.Certificate, 0, 1) + certs, err := tls.LoadX509KeyPair( + path.Join(util.TiDBClientTLSPath, corev1.TLSCertKey), + path.Join(util.TiDBClientTLSPath, corev1.TLSPrivateKeyKey)) + if err != nil { + return "", err + } + clientCert = append(clientCert, certs) + mysql.RegisterTLSConfig("customer", &tls.Config{ + RootCAs: rootCertPool, + Certificates: clientCert, + ServerName: bo.Host, + }) + return fmt.Sprintf("%s:%s@(%s:%d)/%s?tls=customer&charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, constants.TidbMetaDB), nil } func (bo *GenericOptions) GetTikvGCLifeTime(db *sql.DB) (string, error) { diff --git a/docs/api-references/docs.html b/docs/api-references/docs.html index e8fbc31d28..5c050a1663 100644 --- a/docs/api-references/docs.html +++ b/docs/api-references/docs.html @@ -7112,6 +7112,21 @@

TiDBAccessConfig

SecretName is the name of secret which stores tidb cluster’s password.

+ + +tlsClient
+ + +TiDBTLSClient + + + + +(Optional) +

Whether enable the TLS connection between the SQL client and TiDB server +Optional: Defaults to nil

+ +

TiDBConfig @@ -8111,6 +8126,7 @@

TiDBTLSClient

(Appears on: +TiDBAccessConfig, TiDBSpec)

@@ -8149,6 +8165,21 @@

TiDBTLSClient 4. Set Enabled to true.

+ + +tlsSecret
+ +string + + + +(Optional) +

Specify a secret of client cert for backup/restore +Optional: Defaults to -tidb-client-secret +If you want to specify a secret for backup/restore, generate a Secret Object according to the third step of the above procedure, The difference is the Secret Name can be freely defined, and then copy the Secret Name to TLSSecret +this field only work in backup/restore process

+ +

TiKVBlockCacheConfig diff --git a/manifests/backup/backup-aws-s3-br.yaml b/manifests/backup/backup-aws-s3-br.yaml index 1efec57a76..baf00fa088 100644 --- a/manifests/backup/backup-aws-s3-br.yaml +++ b/manifests/backup/backup-aws-s3-br.yaml @@ -13,7 +13,8 @@ spec: br: cluster: myCluster # clusterNamespce: - # enableTLSClient: true + # tlsCluster: + # enabled: false # logLevel: info # statusAddr: # concurrency: 4 @@ -26,6 +27,9 @@ spec: secretName: mySecret # port: 4000 # user: root + # tlsClient: + # enabled: false + # tlsSecret: s3: provider: aws region: us-west-2 diff --git a/manifests/backup/backup-s3-br.yaml b/manifests/backup/backup-s3-br.yaml index d57c23beb7..a99499337e 100644 --- a/manifests/backup/backup-s3-br.yaml +++ b/manifests/backup/backup-s3-br.yaml @@ -13,7 +13,8 @@ spec: br: cluster: myCluster # clusterNamespce: - # enableTLSClient: true + # tlsCluster: + # enabled: false # logLevel: info # statusAddr: # concurrency: 4 @@ -26,6 +27,9 @@ spec: secretName: mySecret # port: 4000 # user: root + # tlsClient: + # enabled: false + # tlsSecret: s3: provider: ceph endpoint: http://10.233.57.220 diff --git a/manifests/backup/backup-schedule-aws-s3-br.yaml b/manifests/backup/backup-schedule-aws-s3-br.yaml index bf9501f784..c66c48a99e 100644 --- a/manifests/backup/backup-schedule-aws-s3-br.yaml +++ b/manifests/backup/backup-schedule-aws-s3-br.yaml @@ -18,7 +18,8 @@ spec: br: cluster: myCluster # clusterNamespce: backupNamespace - # enableTLSClient: true + # tlsCluster: + # enabled: false # logLevel: info # statusAddr: # concurrency: 4 @@ -31,6 +32,9 @@ spec: secretName: mysecret # port: 4000 # user: root + # tlsClient: + # enabled: false + # tlsSecret: s3: provider: aws region: us-west-2 diff --git a/manifests/backup/backup-schedule-s3-br.yaml b/manifests/backup/backup-schedule-s3-br.yaml index 9cfda351e2..14898e84da 100644 --- a/manifests/backup/backup-schedule-s3-br.yaml +++ b/manifests/backup/backup-schedule-s3-br.yaml @@ -18,7 +18,8 @@ spec: br: cluster: myCluster # clusterNamespce: backupNamespace - # enableTLSClient: true + # tlsCluster: + # enabled: false # logLevel: info # statusAddr: # concurrency: 4 @@ -31,6 +32,9 @@ spec: secretName: mysecret # port: 4000 # user: root + # tlsClient: + # enabled: false + # tlsSecret: s3: provider: ceph endpoint: http://10.233.57.220 diff --git a/manifests/backup/restore-aws-s3-br.yaml b/manifests/backup/restore-aws-s3-br.yaml index de5edeedb7..1aea2fdb28 100644 --- a/manifests/backup/restore-aws-s3-br.yaml +++ b/manifests/backup/restore-aws-s3-br.yaml @@ -13,7 +13,8 @@ spec: br: cluster: myCluster # clusterNamespce: - # enableTLSClient: true + # tlsCluster: + # enabled: false # db: # table: # logLevel: info @@ -28,6 +29,9 @@ spec: secretName: mySecret # port: 4000 # user: root + # tlsClient: + # enabled: false + # tlsSecret: s3: provider: aws region: us-west-2 diff --git a/manifests/backup/restore-s3-br.yaml b/manifests/backup/restore-s3-br.yaml index 9ce686f63e..6c86a605f6 100644 --- a/manifests/backup/restore-s3-br.yaml +++ b/manifests/backup/restore-s3-br.yaml @@ -13,7 +13,8 @@ spec: br: cluster: myCluster # clusterNamespce: - # enableTLSClient: true + # tlsCluster: + # enabled: false # db: # table: # logLevel: info @@ -28,6 +29,9 @@ spec: secretName: mySecret # port: 4000 # user: root + # tlsClient: + # enabled: false + # tlsSecret: s3: provider: ceph endpoint: http://10.233.57.220 diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 21ddc555c9..041f2ef936 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -7306,6 +7306,7 @@ spec: description: SecretName is the name of secret which stores tidb cluster's password. type: string + tlsClient: {} user: description: User is the user for login tidb cluster type: string @@ -8242,6 +8243,7 @@ spec: description: SecretName is the name of secret which stores tidb cluster's password. type: string + tlsClient: {} user: description: User is the user for login tidb cluster type: string @@ -9033,6 +9035,7 @@ spec: description: SecretName is the name of secret which stores tidb cluster's password. type: string + tlsClient: {} user: description: User is the user for login tidb cluster type: string diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index f8e4924117..e59ee0bc76 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -3384,10 +3384,18 @@ func schema_pkg_apis_pingcap_v1alpha1_TiDBAccessConfig(ref common.ReferenceCallb Format: "", }, }, + "tlsClient": { + SchemaProps: spec.SchemaProps{ + Description: "Whether enable the TLS connection between the SQL client and TiDB server Optional: Defaults to nil", + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiDBTLSClient"), + }, + }, }, Required: []string{"host", "secretName"}, }, }, + Dependencies: []string{ + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TiDBTLSClient"}, } } diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 5bd0dfed89..30c6967ab3 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -648,6 +648,12 @@ type TiDBTLSClient struct { // 4. Set Enabled to `true`. // +optional Enabled bool `json:"enabled,omitempty"` + // Specify a secret of client cert for backup/restore + // Optional: Defaults to -tidb-client-secret + // +optional + // If you want to specify a secret for backup/restore, generate a Secret Object according to the third step of the above procedure, The difference is the Secret Name can be freely defined, and then copy the Secret Name to TLSSecret + // this field only work in backup/restore process + TLSSecret string `json:"tlsSecret,omitempty"` } // TLSCluster can enable TLS connection between TiDB server components @@ -807,6 +813,10 @@ type TiDBAccessConfig struct { User string `json:"user,omitempty"` // SecretName is the name of secret which stores tidb cluster's password. SecretName string `json:"secretName"` + // Whether enable the TLS connection between the SQL client and TiDB server + // Optional: Defaults to nil + // +optional + TLSClient *TiDBTLSClient `json:"tlsClient,omitempty"` } // +k8s:openapi-gen=true diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index b9b5cab61d..661c1a19d1 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -269,7 +269,7 @@ func (in *BackupScheduleStatus) DeepCopy() *BackupScheduleStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { *out = *in - out.From = in.From + in.From.DeepCopyInto(&out.From) if in.TikvGCLifeTime != nil { in, out := &in.TikvGCLifeTime, &out.TikvGCLifeTime *out = new(string) @@ -2084,7 +2084,7 @@ func (in *RestoreList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) { *out = *in - out.To = in.To + in.To.DeepCopyInto(&out.To) if in.TikvGCLifeTime != nil { in, out := &in.TikvGCLifeTime, &out.TikvGCLifeTime *out = new(string) @@ -2419,6 +2419,11 @@ func (in *TLSCluster) DeepCopy() *TLSCluster { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TiDBAccessConfig) DeepCopyInto(out *TiDBAccessConfig) { *out = *in + if in.TLSClient != nil { + in, out := &in.TLSClient, &out.TLSClient + *out = new(TiDBTLSClient) + **out = **in + } return } diff --git a/pkg/backup/backup/backup_cleaner.go b/pkg/backup/backup/backup_cleaner.go index 001406da15..3f29ab65f6 100644 --- a/pkg/backup/backup/backup_cleaner.go +++ b/pkg/backup/backup/backup_cleaner.go @@ -140,7 +140,7 @@ func (bc *backupCleaner) makeCleanJob(backup *v1alpha1.Backup) (*batchv1.Job, st Name: label.BackupJobLabelVal, Image: controller.TidbBackupManagerImage, Args: args, - ImagePullPolicy: corev1.PullAlways, + ImagePullPolicy: corev1.PullIfNotPresent, Env: storageEnv, }, }, diff --git a/pkg/backup/backup/backup_manager.go b/pkg/backup/backup/backup_manager.go index fa162fba9c..bf10dd5dce 100644 --- a/pkg/backup/backup/backup_manager.go +++ b/pkg/backup/backup/backup_manager.go @@ -211,7 +211,7 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s Name: label.BackupJobLabelVal, Image: controller.TidbBackupManagerImage, Args: args, - ImagePullPolicy: corev1.PullAlways, + ImagePullPolicy: corev1.PullIfNotPresent, VolumeMounts: []corev1.VolumeMount{ {Name: label.BackupJobLabelVal, MountPath: constants.BackupRootPath}, }, @@ -292,6 +292,25 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s }, }) } + if backup.Spec.From.TLSClient != nil && backup.Spec.From.TLSClient.Enabled { + clientSecretName := util.TiDBClientTLSSecretName(backup.Spec.BR.Cluster) + if backup.Spec.From.TLSClient.TLSSecret != "" { + clientSecretName = backup.Spec.From.TLSClient.TLSSecret + } + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "tidb-client-tls", + ReadOnly: true, + MountPath: util.TiDBClientTLSPath, + }) + volumes = append(volumes, corev1.Volume{ + Name: "tidb-client-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: clientSecretName, + }, + }, + }) + } serviceAccount := constants.DefaultServiceAccountName if backup.Spec.ServiceAccount != "" { @@ -309,7 +328,7 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s Name: label.BackupJobLabelVal, Image: controller.TidbBackupManagerImage, Args: args, - ImagePullPolicy: corev1.PullAlways, + ImagePullPolicy: corev1.PullIfNotPresent, VolumeMounts: volumeMounts, Env: envVars, }, diff --git a/pkg/backup/restore/restore_manager.go b/pkg/backup/restore/restore_manager.go index 1a38b7489f..f1407ba3ec 100644 --- a/pkg/backup/restore/restore_manager.go +++ b/pkg/backup/restore/restore_manager.go @@ -197,7 +197,7 @@ func (rm *restoreManager) makeImportJob(restore *v1alpha1.Restore) (*batchv1.Job Name: label.RestoreJobLabelVal, Image: controller.TidbBackupManagerImage, Args: args, - ImagePullPolicy: corev1.PullAlways, + ImagePullPolicy: corev1.PullIfNotPresent, VolumeMounts: []corev1.VolumeMount{ {Name: label.RestoreJobLabelVal, MountPath: constants.BackupRootPath}, }, @@ -277,6 +277,26 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo }) } + if restore.Spec.To.TLSClient != nil && restore.Spec.To.TLSClient.Enabled { + clientSecretName := util.TiDBClientTLSSecretName(restore.Spec.BR.Cluster) + if restore.Spec.To.TLSClient.TLSSecret != "" { + clientSecretName = restore.Spec.To.TLSClient.TLSSecret + } + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "tidb-client-tls", + ReadOnly: true, + MountPath: util.TiDBClientTLSPath, + }) + volumes = append(volumes, corev1.Volume{ + Name: "tidb-client-tls", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: clientSecretName, + }, + }, + }) + } + serviceAccount := constants.DefaultServiceAccountName if restore.Spec.ServiceAccount != "" { serviceAccount = restore.Spec.ServiceAccount @@ -294,7 +314,7 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo Name: label.RestoreJobLabelVal, Image: controller.TidbBackupManagerImage, Args: args, - ImagePullPolicy: corev1.PullAlways, + ImagePullPolicy: corev1.PullIfNotPresent, VolumeMounts: volumeMounts, Env: envVars, },