From 79058d8a4531fcbd873ceb3a850beeec885c0a6e Mon Sep 17 00:00:00 2001 From: pingcap-github-bot Date: Wed, 4 Mar 2020 13:16:32 +0800 Subject: [PATCH] Backup/Restore: support configuring TiKV GC life time (#1835) (#1862) --- cmd/backup-manager/app/backup/backup.go | 10 +- cmd/backup-manager/app/backup/manager.go | 139 ++++++++++++++++- cmd/backup-manager/app/cmd/backup.go | 2 +- cmd/backup-manager/app/cmd/export.go | 12 +- cmd/backup-manager/app/cmd/import.go | 12 +- cmd/backup-manager/app/cmd/restore.go | 4 +- cmd/backup-manager/app/export/export.go | 53 ++----- cmd/backup-manager/app/export/manager.go | 103 ++++++++---- cmd/backup-manager/app/import/manager.go | 44 ++++-- cmd/backup-manager/app/import/restore.go | 27 +--- cmd/backup-manager/app/restore/manager.go | 146 ++++++++++++++++-- cmd/backup-manager/app/restore/restore.go | 10 +- cmd/backup-manager/app/util/generic.go | 60 +++++++ cmd/backup-manager/app/util/util.go | 15 +- images/tidb-backup-manager/Dockerfile | 6 + manifests/crd.yaml | 18 +++ .../pingcap/v1alpha1/openapi_generated.go | 14 ++ pkg/apis/pingcap/v1alpha1/types.go | 8 + .../pingcap/v1alpha1/zz_generated.deepcopy.go | 10 ++ pkg/backup/backup/backup_manager.go | 15 +- pkg/backup/restore/restore_manager.go | 11 +- pkg/backup/util/util.go | 26 ++-- 22 files changed, 546 insertions(+), 199 deletions(-) create mode 100644 cmd/backup-manager/app/util/generic.go diff --git a/cmd/backup-manager/app/backup/backup.go b/cmd/backup-manager/app/backup/backup.go index 51f1d3626d..18cc163a97 100644 --- a/cmd/backup-manager/app/backup/backup.go +++ b/cmd/backup-manager/app/backup/backup.go @@ -20,22 +20,16 @@ import ( "os/exec" "github.com/gogo/protobuf/proto" - "k8s.io/klog" - kvbackup "github.com/pingcap/kvproto/pkg/backup" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "k8s.io/klog" ) // Options contains the input arguments to the backup command type Options struct { - Namespace string - BackupName string -} - -func (bo *Options) String() string { - return fmt.Sprintf("%s/%s", bo.Namespace, bo.BackupName) + util.GenericOptions } func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) { diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index 0fb2543eba..f4389b1d48 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -14,14 +14,19 @@ package backup import ( + "database/sql" "fmt" "time" + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" ) @@ -44,11 +49,29 @@ func NewManager( } } +func (bm *Manager) setOptions(backup *v1alpha1.Backup) { + bm.Options.Host = backup.Spec.From.Host + + if backup.Spec.From.Port != 0 { + bm.Options.Port = backup.Spec.From.Port + } else { + bm.Options.Port = bkconstants.DefaultTidbPort + } + + if backup.Spec.From.User != "" { + bm.Options.User = backup.Spec.From.User + } else { + bm.Options.User = bkconstants.DefaultTidbUser + } + + bm.Options.Password = util.GetOptionValueFromEnv(bkconstants.TidbPasswordKey, bkconstants.BackupManagerEnvVarPrefix) +} + // ProcessBackup used to process the backup logic func (bm *Manager) ProcessBackup() error { - backup, err := bm.backupLister.Backups(bm.Namespace).Get(bm.BackupName) + backup, err := bm.backupLister.Backups(bm.Namespace).Get(bm.ResourceName) if err != nil { - klog.Errorf("can't find cluster %s backup %s CRD object, err: %v", bm, bm.BackupName, err) + klog.Errorf("can't find cluster %s backup %s CRD object, err: %v", bm, bm.ResourceName, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ Type: v1alpha1.BackupFailed, Status: corev1.ConditionTrue, @@ -60,10 +83,34 @@ func (bm *Manager) ProcessBackup() error { if backup.Spec.BR == nil { return fmt.Errorf("no br config in %s", bm) } - return bm.performBackup(backup.DeepCopy()) + + bm.setOptions(backup) + + var db *sql.DB + err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { + db, err = util.OpenDB(bm.GetDSN()) + if err != nil { + klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) + return false, nil + } + return true, nil + }) + + if err != nil { + klog.Errorf("cluster %s connect failed, err: %s", bm, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ConnectTidbFailed", + Message: err.Error(), + }) + } + + defer db.Close() + return bm.performBackup(backup.DeepCopy(), db) } -func (bm *Manager) performBackup(backup *v1alpha1.Backup) error { +func (bm *Manager) performBackup(backup *v1alpha1.Backup, db *sql.DB) error { started := time.Now() err := bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ @@ -74,16 +121,94 @@ func (bm *Manager) performBackup(backup *v1alpha1.Backup) error { return err } - backupFullPath, err := bm.backupData(backup) + oldTikvGCTime, err := bm.GetTikvGCLifeTime(db) if err != nil { - klog.Errorf("backup cluster %s data failed, err: %s", bm, err) + klog.Errorf("cluster %s get %s failed, err: %s", bm, constants.TikvGCVariable, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ Type: v1alpha1.BackupFailed, Status: corev1.ConditionTrue, - Reason: "BackupDataToRemoteFailed", + Reason: "GetTikvGCLifeTimeFailed", Message: err.Error(), }) } + klog.Infof("cluster %s %s is %s", bm, constants.TikvGCVariable, oldTikvGCTime) + + oldTikvGCTimeDuration, err := time.ParseDuration(oldTikvGCTime) + if err != nil { + klog.Errorf("cluster %s parse old %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ParseOldTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + + var tikvGCTimeDuration time.Duration + var tikvGCLifeTime string + if backup.Spec.TikvGCLifeTime != nil { + tikvGCLifeTime = *backup.Spec.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse configured %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ParseConfiguredTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + } else { + tikvGCLifeTime = constants.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse default %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ParseDefaultTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + } + + if oldTikvGCTimeDuration < tikvGCTimeDuration { + err = bm.SetTikvGCLifeTime(db, tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s set tikv GC life time to %s failed, err: %s", bm, tikvGCLifeTime, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "SetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("set cluster %s %s to %s success", bm, constants.TikvGCVariable, tikvGCLifeTime) + } + + backupFullPath, backupErr := bm.backupData(backup) + if oldTikvGCTimeDuration < tikvGCTimeDuration { + err = bm.SetTikvGCLifeTime(db, oldTikvGCTime) + if err != nil { + klog.Errorf("cluster %s reset tikv GC life time to %s failed, err: %s", bm, oldTikvGCTime, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ResetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("reset cluster %s %s to %s success", bm, constants.TikvGCVariable, oldTikvGCTime) + } + if backupErr != nil { + klog.Errorf("backup cluster %s data failed, err: %s", bm, backupErr) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "BackupDataToRemoteFailed", + Message: backupErr.Error(), + }) + } klog.Infof("backup cluster %s data to %s success", bm, backupFullPath) // Note: The size get from remote may be incorrect because the blobs diff --git a/cmd/backup-manager/app/cmd/backup.go b/cmd/backup-manager/app/cmd/backup.go index a67c95dcb1..621103a7be 100644 --- a/cmd/backup-manager/app/cmd/backup.go +++ b/cmd/backup-manager/app/cmd/backup.go @@ -41,7 +41,7 @@ func NewBackupCommand() *cobra.Command { } cmd.Flags().StringVar(&bo.Namespace, "namespace", "", "Backup CR's namespace") - cmd.Flags().StringVar(&bo.BackupName, "backupName", "", "Backup CRD object name") + cmd.Flags().StringVar(&bo.ResourceName, "backupName", "", "Backup CRD object name") return cmd } diff --git a/cmd/backup-manager/app/cmd/export.go b/cmd/backup-manager/app/cmd/export.go index 6cc0c57fb2..f4c19f2d63 100644 --- a/cmd/backup-manager/app/cmd/export.go +++ b/cmd/backup-manager/app/cmd/export.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/export" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" - bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/spf13/cobra" @@ -32,7 +31,7 @@ import ( // NewExportCommand implements the backup command func NewExportCommand() *cobra.Command { - bo := export.BackupOpts{} + bo := export.Options{} cmd := &cobra.Command{ Use: "export", @@ -44,18 +43,13 @@ func NewExportCommand() *cobra.Command { } cmd.Flags().StringVar(&bo.Namespace, "namespace", "", "Backup CR's namespace") - cmd.Flags().StringVar(&bo.Host, "host", "", "Tidb cluster access address") - cmd.Flags().Int32Var(&bo.Port, "port", bkconstants.DefaultTidbPort, "Port number to use for connecting tidb cluster") + cmd.Flags().StringVar(&bo.ResourceName, "backupName", "", "Backup CRD object name") cmd.Flags().StringVar(&bo.Bucket, "bucket", "", "Bucket in which to store the backup data") - cmd.Flags().StringVar(&bo.Password, bkconstants.TidbPasswordKey, "", "Password to use when connecting to tidb cluster") - cmd.Flags().StringVar(&bo.User, "user", "", "User for login tidb cluster") cmd.Flags().StringVar(&bo.StorageType, "storageType", "", "Backend storage type") - cmd.Flags().StringVar(&bo.BackupName, "backupName", "", "Backup CRD object name") - util.SetFlagsFromEnv(cmd.Flags(), bkconstants.BackupManagerEnvVarPrefix) return cmd } -func runExport(backupOpts export.BackupOpts, kubecfg string) error { +func runExport(backupOpts export.Options, kubecfg string) error { kubeCli, cli, err := util.NewKubeAndCRCli(kubecfg) cmdutil.CheckErr(err) options := []informers.SharedInformerOption{ diff --git a/cmd/backup-manager/app/cmd/import.go b/cmd/backup-manager/app/cmd/import.go index 9efbe03946..89a796128a 100644 --- a/cmd/backup-manager/app/cmd/import.go +++ b/cmd/backup-manager/app/cmd/import.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/import" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" - bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/spf13/cobra" @@ -32,7 +31,7 @@ import ( // NewImportCommand implements the restore command func NewImportCommand() *cobra.Command { - ro := _import.RestoreOpts{} + ro := _import.Options{} cmd := &cobra.Command{ Use: "import", @@ -44,17 +43,12 @@ func NewImportCommand() *cobra.Command { } cmd.Flags().StringVar(&ro.Namespace, "namespace", "", "Restore CR's namespace") - cmd.Flags().StringVar(&ro.Host, "host", "", "Tidb cluster access address") - cmd.Flags().Int32Var(&ro.Port, "port", bkconstants.DefaultTidbPort, "Port number to use for connecting tidb cluster") - cmd.Flags().StringVar(&ro.Password, bkconstants.TidbPasswordKey, "", "Password to use when connecting to tidb cluster") - cmd.Flags().StringVar(&ro.User, "user", "", "User for login tidb cluster") - cmd.Flags().StringVar(&ro.RestoreName, "restoreName", "", "Restore CRD object name") + cmd.Flags().StringVar(&ro.ResourceName, "restoreName", "", "Restore CRD object name") cmd.Flags().StringVar(&ro.BackupPath, "backupPath", "", "The location of the backup") - util.SetFlagsFromEnv(cmd.Flags(), bkconstants.BackupManagerEnvVarPrefix) return cmd } -func runImport(restoreOpts _import.RestoreOpts, kubecfg string) error { +func runImport(restoreOpts _import.Options, kubecfg string) error { kubeCli, cli, err := util.NewKubeAndCRCli(kubecfg) cmdutil.CheckErr(err) options := []informers.SharedInformerOption{ diff --git a/cmd/backup-manager/app/cmd/restore.go b/cmd/backup-manager/app/cmd/restore.go index d5835c2ad6..7b6226083b 100644 --- a/cmd/backup-manager/app/cmd/restore.go +++ b/cmd/backup-manager/app/cmd/restore.go @@ -21,7 +21,6 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/restore" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" - bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/spf13/cobra" @@ -44,8 +43,7 @@ func NewRestoreCommand() *cobra.Command { } cmd.Flags().StringVar(&ro.Namespace, "namespace", "", "Restore CR's namespace") - cmd.Flags().StringVar(&ro.RestoreName, "restoreName", "", "Restore CRD object name") - util.SetFlagsFromEnv(cmd.Flags(), bkconstants.BackupManagerEnvVarPrefix) + cmd.Flags().StringVar(&ro.ResourceName, "restoreName", "", "Restore CRD object name") return cmd } diff --git a/cmd/backup-manager/app/export/export.go b/cmd/backup-manager/app/export/export.go index 91e73ca175..ff7a0e19f5 100644 --- a/cmd/backup-manager/app/export/export.go +++ b/cmd/backup-manager/app/export/export.go @@ -14,7 +14,6 @@ package export import ( - "database/sql" "fmt" "io/ioutil" "os/exec" @@ -24,62 +23,32 @@ import ( "time" "github.com/mholt/archiver" - "k8s.io/klog" - "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" + "k8s.io/klog" ) -// BackupOpts contains the input arguments to the backup command -type BackupOpts struct { - Namespace string - BackupName string +// Options contains the input arguments to the backup command +type Options struct { + util.GenericOptions Bucket string - Host string - Port int32 - Password string - User string StorageType string } -func (bo *BackupOpts) String() string { - return fmt.Sprintf("%s/%s", bo.Namespace, bo.BackupName) -} - -func (bo *BackupOpts) getBackupFullPath() string { +func (bo *Options) getBackupFullPath() string { return filepath.Join(constants.BackupRootPath, bo.getBackupRelativePath()) } -func (bo *BackupOpts) getBackupRelativePath() string { +func (bo *Options) getBackupRelativePath() string { backupName := fmt.Sprintf("backup-%s", time.Now().UTC().Format(time.RFC3339)) return fmt.Sprintf("%s/%s", bo.Bucket, backupName) } -func (bo *BackupOpts) getDestBucketURI(remotePath string) string { +func (bo *Options) getDestBucketURI(remotePath string) string { return fmt.Sprintf("%s://%s", bo.StorageType, remotePath) } -func (bo *BackupOpts) getTikvGCLifeTime(db *sql.DB) (string, error) { - var tikvGCTime string - sql := fmt.Sprintf("select variable_value from %s where variable_name= ?", constants.TidbMetaTable) - row := db.QueryRow(sql, constants.TikvGCVariable) - err := row.Scan(&tikvGCTime) - if err != nil { - return tikvGCTime, fmt.Errorf("query cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) - } - return tikvGCTime, nil -} - -func (bo *BackupOpts) setTikvGCLifeTime(db *sql.DB, gcTime string) error { - sql := fmt.Sprintf("update %s set variable_value = ? where variable_name = ?", constants.TidbMetaTable) - _, err := db.Exec(sql, gcTime, constants.TikvGCVariable) - if err != nil { - return fmt.Errorf("set cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) - } - return nil -} - -func (bo *BackupOpts) dumpTidbClusterData() (string, error) { +func (bo *Options) dumpTidbClusterData() (string, error) { bfPath := bo.getBackupFullPath() err := util.EnsureDirectoryExist(bfPath) if err != nil { @@ -105,7 +74,7 @@ func (bo *BackupOpts) dumpTidbClusterData() (string, error) { return bfPath, nil } -func (bo *BackupOpts) backupDataToRemote(source, bucketURI string) error { +func (bo *Options) backupDataToRemote(source, bucketURI string) error { destBucket := util.NormalizeBucketURI(bucketURI) tmpDestBucket := fmt.Sprintf("%s.tmp", destBucket) // TODO: We may need to use exec.CommandContext to control timeouts. @@ -125,10 +94,6 @@ func (bo *BackupOpts) backupDataToRemote(source, bucketURI string) error { return nil } -func (bo *BackupOpts) getDSN(db string) string { - return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, db) -} - /* getCommitTsFromMetadata get commitTs from mydumper's metadata file diff --git a/cmd/backup-manager/app/export/manager.go b/cmd/backup-manager/app/export/manager.go index 22148294e2..b4a488e6ef 100644 --- a/cmd/backup-manager/app/export/manager.go +++ b/cmd/backup-manager/app/export/manager.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" corev1 "k8s.io/api/core/v1" @@ -33,14 +34,14 @@ import ( type BackupManager struct { backupLister listers.BackupLister StatusUpdater controller.BackupConditionUpdaterInterface - BackupOpts + Options } // NewBackupManager return a BackupManager func NewBackupManager( backupLister listers.BackupLister, statusUpdater controller.BackupConditionUpdaterInterface, - backupOpts BackupOpts) *BackupManager { + backupOpts Options) *BackupManager { return &BackupManager{ backupLister, statusUpdater, @@ -48,11 +49,29 @@ func NewBackupManager( } } +func (bm *BackupManager) setOptions(backup *v1alpha1.Backup) { + bm.Options.Host = backup.Spec.From.Host + + if backup.Spec.From.Port != 0 { + bm.Options.Port = backup.Spec.From.Port + } else { + bm.Options.Port = bkconstants.DefaultTidbPort + } + + if backup.Spec.From.User != "" { + bm.Options.User = backup.Spec.From.User + } else { + bm.Options.User = bkconstants.DefaultTidbUser + } + + bm.Options.Password = util.GetOptionValueFromEnv(bkconstants.TidbPasswordKey, bkconstants.BackupManagerEnvVarPrefix) +} + // ProcessBackup used to process the backup logic func (bm *BackupManager) ProcessBackup() error { - backup, err := bm.backupLister.Backups(bm.Namespace).Get(bm.BackupName) + backup, err := bm.backupLister.Backups(bm.Namespace).Get(bm.ResourceName) if err != nil { - klog.Errorf("can't find cluster %s backup %s CRD object, err: %v", bm, bm.BackupName, err) + klog.Errorf("can't find cluster %s backup %s CRD object, err: %v", bm, bm.ResourceName, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ Type: v1alpha1.BackupFailed, Status: corev1.ConditionTrue, @@ -61,15 +80,12 @@ func (bm *BackupManager) ProcessBackup() error { }) } + bm.setOptions(backup) + var db *sql.DB err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err = util.OpenDB(bm.getDSN(constants.TidbMetaDB)) + db, err = util.OpenDB(bm.GetDSN()) if err != nil { - klog.Warningf("can't open connection to tidb cluster %s, err: %v", bm, err) - return false, nil - } - - if err := db.Ping(); err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", bm, err) return false, nil } @@ -101,7 +117,7 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro return err } - oldTikvGCTime, err := bm.getTikvGCLifeTime(db) + oldTikvGCTime, err := bm.GetTikvGCLifeTime(db) if err != nil { klog.Errorf("cluster %s get %s failed, err: %s", bm, constants.TikvGCVariable, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ @@ -123,18 +139,37 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro Message: err.Error(), }) } - tikvGCTimeDuration, err := time.ParseDuration(constants.TikvGCLifeTime) - if err != nil { - klog.Errorf("cluster %s parse default %s failed, err: %s", bm, constants.TikvGCVariable, err) - return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ - Type: v1alpha1.BackupFailed, - Status: corev1.ConditionTrue, - Reason: "ParseDefaultTikvGCLifeTimeFailed", - Message: err.Error(), - }) + + var tikvGCTimeDuration time.Duration + var tikvGCLifeTime string + if backup.Spec.TikvGCLifeTime != nil { + tikvGCLifeTime = *backup.Spec.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse configured %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ParseConfiguredTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + } else { + tikvGCLifeTime = constants.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse default %s failed, err: %s", bm, constants.TikvGCVariable, err) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "ParseDefaultTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } } + if oldTikvGCTimeDuration < tikvGCTimeDuration { - err = bm.setTikvGCLifeTime(db, constants.TikvGCLifeTime) + err = bm.SetTikvGCLifeTime(db, constants.TikvGCLifeTime) if err != nil { klog.Errorf("cluster %s set tikv GC life time to %s failed, err: %s", bm, constants.TikvGCLifeTime, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ @@ -147,20 +182,9 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro klog.Infof("set cluster %s %s to %s success", bm, constants.TikvGCVariable, constants.TikvGCLifeTime) } - backupFullPath, err := bm.dumpTidbClusterData() - if err != nil { - klog.Errorf("dump cluster %s data failed, err: %s", bm, err) - return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ - Type: v1alpha1.BackupFailed, - Status: corev1.ConditionTrue, - Reason: "DumpTidbClusterFailed", - Message: err.Error(), - }) - } - klog.Infof("dump cluster %s data to %s success", bm, backupFullPath) - + backupFullPath, backupErr := bm.dumpTidbClusterData() if oldTikvGCTimeDuration < tikvGCTimeDuration { - err = bm.setTikvGCLifeTime(db, oldTikvGCTime) + err = bm.SetTikvGCLifeTime(db, oldTikvGCTime) if err != nil { klog.Errorf("cluster %s reset tikv GC life time to %s failed, err: %s", bm, oldTikvGCTime, err) return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ @@ -172,6 +196,17 @@ func (bm *BackupManager) performBackup(backup *v1alpha1.Backup, db *sql.DB) erro } klog.Infof("reset cluster %s %s to %s success", bm, constants.TikvGCVariable, oldTikvGCTime) } + if backupErr != nil { + klog.Errorf("dump cluster %s data failed, err: %s", bm, backupErr) + return bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{ + Type: v1alpha1.BackupFailed, + Status: corev1.ConditionTrue, + Reason: "DumpTidbClusterFailed", + Message: backupErr.Error(), + }) + } + klog.Infof("dump cluster %s data to %s success", bm, backupFullPath) + // TODO: Concurrent get file size and upload backup data to speed up processing time archiveBackupPath := backupFullPath + constants.DefaultArchiveExtention err = archiveBackupData(backupFullPath, archiveBackupPath) diff --git a/cmd/backup-manager/app/import/manager.go b/cmd/backup-manager/app/import/manager.go index cc17881052..3d6ac2ce31 100644 --- a/cmd/backup-manager/app/import/manager.go +++ b/cmd/backup-manager/app/import/manager.go @@ -14,6 +14,7 @@ package _import import ( + "database/sql" "fmt" "path/filepath" "time" @@ -21,6 +22,7 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" corev1 "k8s.io/api/core/v1" @@ -29,30 +31,48 @@ import ( "k8s.io/klog" ) -// RestoreManager mainly used to manage backup related work +// RestoreManager mainly used to manage restore related work type RestoreManager struct { restoreLister listers.RestoreLister StatusUpdater controller.RestoreConditionUpdaterInterface - RestoreOpts + Options } // NewRestoreManager return a RestoreManager func NewRestoreManager( restoreLister listers.RestoreLister, statusUpdater controller.RestoreConditionUpdaterInterface, - backupOpts RestoreOpts) *RestoreManager { + restoreOpts Options) *RestoreManager { return &RestoreManager{ restoreLister, statusUpdater, - backupOpts, + restoreOpts, } } +func (rm *RestoreManager) setOptions(restore *v1alpha1.Restore) { + rm.Options.Host = restore.Spec.To.Host + + if restore.Spec.To.Port != 0 { + rm.Options.Port = restore.Spec.To.Port + } else { + rm.Options.Port = bkconstants.DefaultTidbPort + } + + if restore.Spec.To.User != "" { + rm.Options.User = restore.Spec.To.User + } else { + rm.Options.User = bkconstants.DefaultTidbUser + } + + rm.Options.Password = util.GetOptionValueFromEnv(bkconstants.TidbPasswordKey, bkconstants.BackupManagerEnvVarPrefix) +} + // ProcessRestore used to process the restore logic func (rm *RestoreManager) ProcessRestore() error { - restore, err := rm.restoreLister.Restores(rm.Namespace).Get(rm.RestoreName) + restore, err := rm.restoreLister.Restores(rm.Namespace).Get(rm.ResourceName) if err != nil { - klog.Errorf("can't find cluster %s restore %s CRD object, err: %v", rm, rm.RestoreName, err) + klog.Errorf("can't find cluster %s restore %s CRD object, err: %v", rm, rm.ResourceName, err) return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ Type: v1alpha1.RestoreFailed, Status: corev1.ConditionTrue, @@ -61,18 +81,15 @@ func (rm *RestoreManager) ProcessRestore() error { }) } + rm.setOptions(restore) + + var db *sql.DB err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { - db, err := util.OpenDB(rm.getDSN(constants.TidbMetaDB)) + db, err = util.OpenDB(rm.GetDSN()) if err != nil { - klog.Warningf("can't open connection to tidb cluster %s, err: %v", rm, err) - return false, nil - } - - if err := db.Ping(); err != nil { klog.Warningf("can't connect to tidb cluster %s, err: %s", rm, err) return false, nil } - db.Close() return true, nil }) @@ -86,6 +103,7 @@ func (rm *RestoreManager) ProcessRestore() error { }) } + defer db.Close() return rm.performRestore(restore.DeepCopy()) } diff --git a/cmd/backup-manager/app/import/restore.go b/cmd/backup-manager/app/import/restore.go index 9db1c30a6d..dfcee888c4 100644 --- a/cmd/backup-manager/app/import/restore.go +++ b/cmd/backup-manager/app/import/restore.go @@ -24,28 +24,19 @@ import ( "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" ) -// RestoreOpts contains the input arguments to the restore command -type RestoreOpts struct { - Namespace string - RestoreName string - Password string - Host string - Port int32 - User string - BackupPath string +// Options contains the input arguments to the restore command +type Options struct { + util.GenericOptions + BackupPath string } -func (ro *RestoreOpts) String() string { - return fmt.Sprintf("%s/%s", ro.Namespace, ro.RestoreName) -} - -func (ro *RestoreOpts) getRestoreDataPath() string { +func (ro *Options) getRestoreDataPath() string { backupName := filepath.Base(ro.BackupPath) bucketName := filepath.Base(filepath.Dir(ro.BackupPath)) return filepath.Join(constants.BackupRootPath, bucketName, backupName) } -func (ro *RestoreOpts) downloadBackupData(localPath string) error { +func (ro *Options) downloadBackupData(localPath string) error { if err := util.EnsureDirectoryExist(filepath.Dir(localPath)); err != nil { return err } @@ -62,7 +53,7 @@ func (ro *RestoreOpts) downloadBackupData(localPath string) error { return nil } -func (ro *RestoreOpts) loadTidbClusterData(restorePath string) error { +func (ro *Options) loadTidbClusterData(restorePath string) error { if exist := util.IsDirExist(restorePath); !exist { return fmt.Errorf("dir %s does not exist or is not a dir", restorePath) } @@ -81,10 +72,6 @@ func (ro *RestoreOpts) loadTidbClusterData(restorePath string) error { return nil } -func (ro *RestoreOpts) getDSN(db string) string { - return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", ro.User, ro.Password, ro.Host, ro.Port, db) -} - // unarchiveBackupData unarchive backup data to dest dir func unarchiveBackupData(backupFile, destDir string) (string, error) { var unarchiveBackupPath string diff --git a/cmd/backup-manager/app/restore/manager.go b/cmd/backup-manager/app/restore/manager.go index ec034badcc..56b06b66f5 100644 --- a/cmd/backup-manager/app/restore/manager.go +++ b/cmd/backup-manager/app/restore/manager.go @@ -14,16 +14,20 @@ package restore import ( + "database/sql" "fmt" "time" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog" - + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + bkconstants "github.com/pingcap/tidb-operator/pkg/backup/constants" listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" ) type Manager struct { @@ -44,11 +48,29 @@ func NewManager( } } +func (rm *Manager) setOptions(restore *v1alpha1.Restore) { + rm.Options.Host = restore.Spec.To.Host + + if restore.Spec.To.Port != 0 { + rm.Options.Port = restore.Spec.To.Port + } else { + rm.Options.Port = bkconstants.DefaultTidbPort + } + + if restore.Spec.To.User != "" { + rm.Options.User = restore.Spec.To.User + } else { + rm.Options.User = bkconstants.DefaultTidbUser + } + + rm.Options.Password = util.GetOptionValueFromEnv(bkconstants.TidbPasswordKey, bkconstants.BackupManagerEnvVarPrefix) +} + // ProcessRestore used to process the restore logic func (rm *Manager) ProcessRestore() error { - restore, err := rm.restoreLister.Restores(rm.Namespace).Get(rm.RestoreName) + restore, err := rm.restoreLister.Restores(rm.Namespace).Get(rm.ResourceName) if err != nil { - klog.Errorf("can't find cluster %s restore %s CRD object, err: %v", rm, rm.RestoreName, err) + klog.Errorf("can't find cluster %s restore %s CRD object, err: %v", rm, rm.ResourceName, err) return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ Type: v1alpha1.RestoreFailed, Status: corev1.ConditionTrue, @@ -60,10 +82,33 @@ func (rm *Manager) ProcessRestore() error { return fmt.Errorf("no br config in %s", rm) } - return rm.performRestore(restore.DeepCopy()) + rm.setOptions(restore) + + var db *sql.DB + err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) { + db, err = util.OpenDB(rm.GetDSN()) + if err != nil { + klog.Warningf("can't connect to tidb cluster %s, err: %s", rm, err) + return false, nil + } + return true, nil + }) + + if err != nil { + klog.Errorf("cluster %s connect failed, err: %s", rm, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ConnectTidbFailed", + Message: err.Error(), + }) + } + + defer db.Close() + return rm.performRestore(restore.DeepCopy(), db) } -func (rm *Manager) performRestore(restore *v1alpha1.Restore) error { +func (rm *Manager) performRestore(restore *v1alpha1.Restore, db *sql.DB) error { started := time.Now() err := rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ @@ -74,15 +119,94 @@ func (rm *Manager) performRestore(restore *v1alpha1.Restore) error { return err } - if err := rm.restoreData(restore); err != nil { - klog.Errorf("restore cluster %s from %s failed, err: %s", rm, restore.Spec.Type, err) + oldTikvGCTime, err := rm.GetTikvGCLifeTime(db) + if err != nil { + klog.Errorf("cluster %s get %s failed, err: %s", rm, constants.TikvGCVariable, err) return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ Type: v1alpha1.RestoreFailed, Status: corev1.ConditionTrue, - Reason: "RestoreDataFromRemoteFailed", + Reason: "GetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("cluster %s %s is %s", rm, constants.TikvGCVariable, oldTikvGCTime) + + oldTikvGCTimeDuration, err := time.ParseDuration(oldTikvGCTime) + if err != nil { + klog.Errorf("cluster %s parse old %s failed, err: %s", rm, constants.TikvGCVariable, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ParseOldTikvGCLifeTimeFailed", Message: err.Error(), }) } + + var tikvGCTimeDuration time.Duration + var tikvGCLifeTime string + if restore.Spec.TikvGCLifeTime != nil { + tikvGCLifeTime = *restore.Spec.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse configured %s failed, err: %s", rm, constants.TikvGCVariable, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ParseConfiguredTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + } else { + tikvGCLifeTime = constants.TikvGCLifeTime + tikvGCTimeDuration, err = time.ParseDuration(tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s parse default %s failed, err: %s", rm, constants.TikvGCVariable, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ParseDefaultTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + } + + if oldTikvGCTimeDuration < tikvGCTimeDuration { + err = rm.SetTikvGCLifeTime(db, tikvGCLifeTime) + if err != nil { + klog.Errorf("cluster %s set tikv GC life time to %s failed, err: %s", rm, tikvGCLifeTime, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "SetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("set cluster %s %s to %s success", rm, constants.TikvGCVariable, tikvGCLifeTime) + } + + restoreErr := rm.restoreData(restore) + if oldTikvGCTimeDuration < tikvGCTimeDuration { + err = rm.SetTikvGCLifeTime(db, oldTikvGCTime) + if err != nil { + klog.Errorf("cluster %s reset tikv GC life time to %s failed, err: %s", rm, oldTikvGCTime, err) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "ResetTikvGCLifeTimeFailed", + Message: err.Error(), + }) + } + klog.Infof("reset cluster %s %s to %s success", rm, constants.TikvGCVariable, oldTikvGCTime) + } + if restoreErr != nil { + klog.Errorf("restore cluster %s from %s failed, err: %s", rm, restore.Spec.Type, restoreErr) + return rm.StatusUpdater.Update(restore, &v1alpha1.RestoreCondition{ + Type: v1alpha1.RestoreFailed, + Status: corev1.ConditionTrue, + Reason: "RestoreDataFromRemoteFailed", + Message: restoreErr.Error(), + }) + } klog.Infof("restore cluster %s from %s succeed", rm, restore.Spec.Type) finish := time.Now() diff --git a/cmd/backup-manager/app/restore/restore.go b/cmd/backup-manager/app/restore/restore.go index fac9d5e4f7..347bebcb43 100644 --- a/cmd/backup-manager/app/restore/restore.go +++ b/cmd/backup-manager/app/restore/restore.go @@ -17,19 +17,13 @@ import ( "fmt" "os/exec" - "k8s.io/klog" - "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "k8s.io/klog" ) type Options struct { - Namespace string - RestoreName string -} - -func (ro *Options) String() string { - return fmt.Sprintf("%s/%s", ro.Namespace, ro.RestoreName) + util.GenericOptions } func (ro *Options) restoreData(restore *v1alpha1.Restore) error { diff --git a/cmd/backup-manager/app/util/generic.go b/cmd/backup-manager/app/util/generic.go new file mode 100644 index 0000000000..04bebc5b3b --- /dev/null +++ b/cmd/backup-manager/app/util/generic.go @@ -0,0 +1,60 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "database/sql" + "fmt" + + "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" +) + +// GenericOptions contains the generic input arguments to the backup/restore command +type GenericOptions struct { + Namespace string + // ResourceName can be the name of a backup or restore resource + ResourceName string + Host string + Port int32 + Password string + User string +} + +func (bo *GenericOptions) String() string { + return fmt.Sprintf("%s/%s", bo.Namespace, bo.ResourceName) +} + +func (bo *GenericOptions) GetDSN() string { + return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", bo.User, bo.Password, bo.Host, bo.Port, constants.TidbMetaDB) +} + +func (bo *GenericOptions) GetTikvGCLifeTime(db *sql.DB) (string, error) { + var tikvGCTime string + sql := fmt.Sprintf("select variable_value from %s where variable_name= ?", constants.TidbMetaTable) + row := db.QueryRow(sql, constants.TikvGCVariable) + err := row.Scan(&tikvGCTime) + if err != nil { + return tikvGCTime, fmt.Errorf("query cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) + } + return tikvGCTime, nil +} + +func (bo *GenericOptions) SetTikvGCLifeTime(db *sql.DB, gcTime string) error { + sql := fmt.Sprintf("update %s set variable_value = ? where variable_name = ?", constants.TidbMetaTable) + _, err := db.Exec(sql, gcTime, constants.TikvGCVariable) + if err != nil { + return fmt.Errorf("set cluster %s %s failed, sql: %s, err: %v", bo, constants.TikvGCVariable, sql, err) + } + return nil +} diff --git a/cmd/backup-manager/app/util/util.go b/cmd/backup-manager/app/util/util.go index e7d436c14c..093b09e8b3 100644 --- a/cmd/backup-manager/app/util/util.go +++ b/cmd/backup-manager/app/util/util.go @@ -98,17 +98,10 @@ func NormalizeBucketURI(bucket string) string { return strings.Replace(bucket, "://", ":", 1) } -// SetFlagsFromEnv set the environment variable. Will override default values, but be overridden by command line parameters. -func SetFlagsFromEnv(flags *pflag.FlagSet, prefix string) error { - flags.VisitAll(func(f *pflag.Flag) { - envVar := prefix + "_" + strings.Replace(strings.ToUpper(f.Name), "-", "_", -1) - value := os.Getenv(envVar) - if value != "" { - flags.Set(f.Name, value) - } - }) - - return nil +// GetOptionValueFromEnv get option's value from environment variable. If unset, return empty string. +func GetOptionValueFromEnv(option, envPrefix string) string { + envVar := envPrefix + "_" + strings.Replace(strings.ToUpper(option), "-", "_", -1) + return os.Getenv(envVar) } // ConstructBRGlobalOptionsForBackup constructs BR global options for backup and also return the remote path. diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile index 32f0940baa..861490bb88 100644 --- a/images/tidb-backup-manager/Dockerfile +++ b/images/tidb-backup-manager/Dockerfile @@ -9,6 +9,12 @@ RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-$ && chmod 755 /usr/local/bin/rclone \ && rm -rf rclone-${VERSION}-linux-amd64.zip rclone-${VERSION}-linux-amd64 +RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \ + && tar -xzf br-latest-linux-amd64.tar.gz \ + && mv bin/br /usr/local/bin \ + && chmod 755 /usr/local/bin/br \ + && rm -rf br-latest-linux-amd64.tar.gz + COPY bin/tidb-backup-manager /tidb-backup-manager COPY entrypoint.sh /entrypoint.sh diff --git a/manifests/crd.yaml b/manifests/crd.yaml index ac884e8801..d9473efafb 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -6907,6 +6907,12 @@ spec: storageSize: description: StorageSize is the request storage size for backup job type: string + tikvGCLifeTime: + description: TikvGCLifeTime is to specify the safe gc life time for + backup. The time limit during which data is retained for each GC, + in the format of Go Duration. When a GC happens, the current time + minus this value is the safe point. + type: string tolerations: description: Base tolerations of backup Pods, components may add more tolerations upon this respectively @@ -7720,6 +7726,12 @@ spec: storageSize: description: StorageSize is the request storage size for backup job type: string + tikvGCLifeTime: + description: TikvGCLifeTime is to specify the safe gc life time for + restore. The time limit during which data is retained for each GC, + in the format of Go Duration. When a GC happens, the current time + minus this value is the safe point. + type: string to: description: TiDBAccessConfig defines the configuration for access tidb cluster @@ -8623,6 +8635,12 @@ spec: description: StorageSize is the request storage size for backup job type: string + tikvGCLifeTime: + description: TikvGCLifeTime is to specify the safe gc life time + for backup. The time limit during which data is retained for each + GC, in the format of Go Duration. When a GC happens, the current + time minus this value is the safe point. + type: string tolerations: description: Base tolerations of backup Pods, components may add more tolerations upon this respectively diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index f1384879c6..a3afc95461 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -723,6 +723,13 @@ func schema_pkg_apis_pingcap_v1alpha1_BackupSpec(ref common.ReferenceCallback) c Format: "", }, }, + "tikvGCLifeTime": { + SchemaProps: spec.SchemaProps{ + Description: "TikvGCLifeTime is to specify the safe gc life time for backup. The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point.", + Type: []string{"string"}, + Format: "", + }, + }, "s3": { SchemaProps: spec.SchemaProps{ Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.S3StorageProvider"), @@ -2787,6 +2794,13 @@ func schema_pkg_apis_pingcap_v1alpha1_RestoreSpec(ref common.ReferenceCallback) Format: "", }, }, + "tikvGCLifeTime": { + SchemaProps: spec.SchemaProps{ + Description: "TikvGCLifeTime is to specify the safe gc life time for restore. The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point.", + Type: []string{"string"}, + Format: "", + }, + }, "s3": { SchemaProps: spec.SchemaProps{ Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.S3StorageProvider"), diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index 80b26c0f5c..1a8ffbe3a1 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -765,6 +765,10 @@ type BackupSpec struct { From TiDBAccessConfig `json:"from,omitempty"` // Type is the backup type for tidb cluster. Type BackupType `json:"backupType,omitempty"` + // TikvGCLifeTime is to specify the safe gc life time for backup. + // The time limit during which data is retained for each GC, in the format of Go Duration. + // When a GC happens, the current time minus this value is the safe point. + TikvGCLifeTime *string `json:"tikvGCLifeTime,omitempty"` // StorageProvider configures where and how backups should be stored. StorageProvider `json:",inline"` // The storageClassName of the persistent volume for Backup data storage. @@ -981,6 +985,10 @@ type RestoreSpec struct { To TiDBAccessConfig `json:"to,omitempty"` // Type is the backup type for tidb cluster. Type BackupType `json:"backupType,omitempty"` + // TikvGCLifeTime is to specify the safe gc life time for restore. + // The time limit during which data is retained for each GC, in the format of Go Duration. + // When a GC happens, the current time minus this value is the safe point. + TikvGCLifeTime *string `json:"tikvGCLifeTime,omitempty"` // StorageProvider configures where and how backups should be stored. StorageProvider `json:",inline"` // The storageClassName of the persistent volume for Restore data storage. diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index 2dda9cf4cd..ecb5c5ec15 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -265,6 +265,11 @@ func (in *BackupScheduleStatus) DeepCopy() *BackupScheduleStatus { func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { *out = *in out.From = in.From + if in.TikvGCLifeTime != nil { + in, out := &in.TikvGCLifeTime, &out.TikvGCLifeTime + *out = new(string) + **out = **in + } in.StorageProvider.DeepCopyInto(&out.StorageProvider) if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName @@ -1915,6 +1920,11 @@ func (in *RestoreList) DeepCopyObject() runtime.Object { func (in *RestoreSpec) DeepCopyInto(out *RestoreSpec) { *out = *in out.To = in.To + if in.TikvGCLifeTime != nil { + in, out := &in.TikvGCLifeTime, &out.TikvGCLifeTime + *out = new(string) + **out = **in + } in.StorageProvider.DeepCopyInto(&out.StorageProvider) if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName diff --git a/pkg/backup/backup/backup_manager.go b/pkg/backup/backup/backup_manager.go index 9cbf246041..3922099e3b 100644 --- a/pkg/backup/backup/backup_manager.go +++ b/pkg/backup/backup/backup_manager.go @@ -188,11 +188,8 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s args := []string{ "export", fmt.Sprintf("--namespace=%s", ns), - fmt.Sprintf("--host=%s", backup.Spec.From.Host), - fmt.Sprintf("--port=%d", backup.Spec.From.Port), - fmt.Sprintf("--user=%s", backup.Spec.From.User), - fmt.Sprintf("--bucket=%s", bucketName), fmt.Sprintf("--backupName=%s", name), + fmt.Sprintf("--bucket=%s", bucketName), fmt.Sprintf("--storageType=%s", backuputil.GetStorageType(backup.Spec.StorageProvider)), } @@ -250,15 +247,23 @@ func (bm *backupManager) makeExportJob(backup *v1alpha1.Backup) (*batchv1.Job, s return job, "", nil } + func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, string, error) { ns := backup.GetNamespace() name := backup.GetName() - envVars, reason, err := backuputil.GenerateStorageCertEnv(ns, backup.Spec.StorageProvider, bm.secretLister) + envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, backup.Spec.From.SecretName, bm.secretLister) + if err != nil { + return nil, reason, err + } + + storageEnv, reason, err := backuputil.GenerateStorageCertEnv(ns, backup.Spec.StorageProvider, bm.secretLister) if err != nil { return nil, reason, fmt.Errorf("backup %s/%s, %v", ns, name, err) } + envVars = append(envVars, storageEnv...) + args := []string{ "backup", fmt.Sprintf("--namespace=%s", ns), diff --git a/pkg/backup/restore/restore_manager.go b/pkg/backup/restore/restore_manager.go index 619d719e0f..1dbd11b94f 100644 --- a/pkg/backup/restore/restore_manager.go +++ b/pkg/backup/restore/restore_manager.go @@ -174,9 +174,6 @@ func (rm *restoreManager) makeImportJob(restore *v1alpha1.Restore) (*batchv1.Job "import", fmt.Sprintf("--namespace=%s", ns), fmt.Sprintf("--restoreName=%s", name), - fmt.Sprintf("--host=%s", restore.Spec.To.Host), - fmt.Sprintf("--port=%d", restore.Spec.To.Port), - fmt.Sprintf("--user=%s", restore.Spec.To.User), fmt.Sprintf("--backupPath=%s", backupPath), } @@ -238,11 +235,17 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo ns := restore.GetNamespace() name := restore.GetName() - envVars, reason, err := backuputil.GenerateStorageCertEnv(ns, restore.Spec.StorageProvider, rm.secretLister) + envVars, reason, err := backuputil.GenerateTidbPasswordEnv(ns, name, restore.Spec.To.SecretName, rm.secretLister) + if err != nil { + return nil, reason, err + } + + storageEnv, reason, err := backuputil.GenerateStorageCertEnv(ns, restore.Spec.StorageProvider, rm.secretLister) if err != nil { return nil, reason, fmt.Errorf("restore %s/%s, %v", ns, name, err) } + envVars = append(envVars, storageEnv...) args := []string{ "restore", fmt.Sprintf("--namespace=%s", ns), diff --git a/pkg/backup/util/util.go b/pkg/backup/util/util.go index b22ebdefad..d62307b7bb 100644 --- a/pkg/backup/util/util.go +++ b/pkg/backup/util/util.go @@ -287,13 +287,14 @@ func GetBackupDataPath(provider v1alpha1.StorageProvider) (string, string, error func ValidateBackup(backup *v1alpha1.Backup) error { ns := backup.Namespace name := backup.Name + + if backup.Spec.From.Host == "" { + return fmt.Errorf("missing cluster config in spec of %s/%s", ns, name) + } + if backup.Spec.From.SecretName == "" { + return fmt.Errorf("missing tidbSecretName config in spec of %s/%s", ns, name) + } if backup.Spec.BR == nil { - if backup.Spec.From.Host == "" { - return fmt.Errorf("missing cluster config in spec of %s/%s", ns, name) - } - if backup.Spec.From.SecretName == "" { - return fmt.Errorf("missing tidbSecretName config in spec of %s/%s", ns, name) - } if backup.Spec.StorageSize == "" { return fmt.Errorf("missing StorageSize config in spec of %s/%s", ns, name) } @@ -338,13 +339,14 @@ func ValidateBackup(backup *v1alpha1.Backup) error { func ValidateRestore(restore *v1alpha1.Restore) error { ns := restore.Namespace name := restore.Name + + if restore.Spec.To.Host == "" { + return fmt.Errorf("missing cluster config in spec of %s/%s", ns, name) + } + if restore.Spec.To.SecretName == "" { + return fmt.Errorf("missing tidbSecretName config in spec of %s/%s", ns, name) + } if restore.Spec.BR == nil { - if restore.Spec.To.Host == "" { - return fmt.Errorf("missing cluster config in spec of %s/%s", ns, name) - } - if restore.Spec.To.SecretName == "" { - return fmt.Errorf("missing tidbSecretName config in spec of %s/%s", ns, name) - } if restore.Spec.StorageSize == "" { return fmt.Errorf("missing StorageSize config in spec of %s/%s", ns, name) }