Merge branch 'master' into short-labels

cmd/backup-manager/app/cmd/restore.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants"
 	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/restore"
 	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/util"
+	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions"
 	"github.com/pingcap/tidb-operator/pkg/controller"
 	"github.com/spf13/cobra"
@@ -48,6 +49,8 @@ func NewRestoreCommand() *cobra.Command {
 	cmd.Flags().BoolVar(&ro.TLSClient, "client-tls", false, "Whether client tls is enabled")
 	cmd.Flags().BoolVar(&ro.TLSCluster, "cluster-tls", false, "Whether cluster tls is enabled")
 	cmd.Flags().BoolVar(&ro.SkipClientCA, "skipClientCA", false, "Whether to skip tidb server's certificates validation")
+	cmd.Flags().StringVar(&ro.Mode, "mode", string(v1alpha1.RestoreModeSnapshot), "restore mode, which is pitr or snapshot(default)")
+	cmd.Flags().StringVar(&ro.PitrRestoredTs, "pitrRestoredTs", "0", "The pitr restored ts")
 	return cmd
 }
 

cmd/backup-manager/app/restore/restore.go

@@ -59,6 +59,21 @@ func (ro *Options) restoreData(ctx context.Context, restore *v1alpha1.Restore) e
 	} else {
 		restoreType = string(restore.Spec.Type)
 	}
+
+	// gen PiTR args
+	if ro.Mode == string(v1alpha1.RestoreModePiTR) {
+		// init pitr restore args
+		args = append(args, fmt.Sprintf("--restored-ts=%s", ro.PitrRestoredTs))
+
+		if fullBackupArgs, err := backupUtil.GenStorageArgsForFlag(restore.Spec.PitrFullBackupStorageProvider, "full-backup-storage"); err != nil {
+			return err
+		} else {
+			// parse full backup path
+			args = append(args, fullBackupArgs...)
+		}
+		restoreType = "point"
+	}
+
 	fullArgs := []string{
 		"restore",
 		restoreType,

cmd/backup-manager/app/util/generic.go

@@ -33,19 +33,20 @@ import (
 type GenericOptions struct {
 	Namespace string
 	// ResourceName can be the name of a backup or restore resource
-	ResourceName  string
-	TLSClient     bool
-	TLSCluster    bool
-	SkipClientCA  bool
-	Host          string
-	Port          int32
-	Password      string
-	User          string
-	TiKVVersion   string
-	Mode          string
-	SubCommand    string
-	CommitTS      string
-	TruncateUntil string
+	ResourceName   string
+	TLSClient      bool
+	TLSCluster     bool
+	SkipClientCA   bool
+	Host           string
+	Port           int32
+	Password       string
+	User           string
+	TiKVVersion    string
+	Mode           string
+	SubCommand     string
+	CommitTS       string
+	TruncateUntil  string
+	PitrRestoredTs string
 }
 
 func (bo *GenericOptions) String() string {

cmd/backup-manager/app/util/remote.go

@@ -46,7 +46,8 @@ import (
 )
 
 const (
-	maxRetries = 3 // number of retries to make of operations
+	maxRetries         = 3 // number of retries to make of operations
+	defaultStorageFlag = "storage"
 )
 
 type s3Config struct {
@@ -314,26 +315,26 @@ func BatchDeleteObjectsConcurrently(ctx context.Context, bucket *blob.Bucket, ob
 	return result
 }
 
-// genStorageArgs returns the arg for --storage option and the remote/local path for br
+// genStorageArgs returns the arg for --flag option and the remote/local path for br, default flag is storage.
 // TODO: add unit test
-func genStorageArgs(provider v1alpha1.StorageProvider) ([]string, error) {
+func GenStorageArgsForFlag(provider v1alpha1.StorageProvider, flag string) ([]string, error) {
 	st := util.GetStorageType(provider)
 	switch st {
 	case v1alpha1.BackupStorageTypeS3:
 		qs := makeS3Config(provider.S3, false)
-		s := newS3StorageOption(qs)
+		s := newS3StorageOptionForFlag(qs, flag)
 		return s, nil
 	case v1alpha1.BackupStorageTypeGcs:
 		qs := makeGcsConfig(provider.Gcs, false)
-		s := newGcsStorageOption(qs)
+		s := newGcsStorageOptionForFlag(qs, flag)
 		return s, nil
 	case v1alpha1.BackupStorageTypeAzblob:
 		conf := makeAzblobConfig(provider.Azblob)
-		strs := newAzblobStorageOption(conf)
+		strs := newAzblobStorageOptionForFlag(conf, flag)
 		return strs, nil
 	case v1alpha1.BackupStorageTypeLocal:
 		localConfig := makeLocalConfig(provider.Local)
-		cmdOpts, err := newLocalStorageOption(localConfig)
+		cmdOpts, err := newLocalStorageOptionForFlag(localConfig, flag)
 		if err != nil {
 			return nil, err
 		}
@@ -343,16 +344,26 @@ func genStorageArgs(provider v1alpha1.StorageProvider) ([]string, error) {
 	}
 }
 
-// newLocalStorageOption constructs `--storage local://$PATH` arg for br
-func newLocalStorageOption(conf *localConfig) ([]string, error) {
+// newLocalStorageOption constructs `--flag local://$PATH` arg for br
+func newLocalStorageOptionForFlag(conf *localConfig, flag string) ([]string, error) {
+	if flag != "" && flag != defaultStorageFlag {
+		// now just set path to special flag
+		return []string{fmt.Sprintf("--%s=local://%s", flag, path.Join(conf.mountPath, conf.prefix))}, nil
+	}
 	return []string{fmt.Sprintf("--storage=local://%s", path.Join(conf.mountPath, conf.prefix))}, nil
 }
 
-// newS3StorageOption constructs the arg for --storage option and the remote path for br
-func newS3StorageOption(conf *s3Config) []string {
+// newS3StorageOption constructs the arg for --flag option and the remote path for br
+func newS3StorageOptionForFlag(conf *s3Config, flag string) []string {
 	var s3options []string
 	path := fmt.Sprintf("s3://%s", path.Join(conf.bucket, conf.prefix))
+	if flag != "" && flag != defaultStorageFlag {
+		// now just set path to special flag
+		s3options = append(s3options, fmt.Sprintf("--%s=%s", flag, path))
+		return s3options
+	}
 	s3options = append(s3options, fmt.Sprintf("--storage=%s", path))
+
 	if conf.region != "" {
 		s3options = append(s3options, fmt.Sprintf("--s3.region=%s", conf.region))
 	}
@@ -548,11 +559,17 @@ func newAzblobStorageUsingSharedKey(conf *azblobConfig, cred *azblobSharedKeyCre
 	return azureblob.OpenBucket(ctx, pipeline, accountName, conf.container, &azureblob.Options{Credential: credential})
 }
 
-// newGcsStorageOption constructs the arg for --storage option and the remote path for br
-func newGcsStorageOption(conf *gcsConfig) []string {
+// newGcsStorageOption constructs the arg for --flag option and the remote path for br
+func newGcsStorageOptionForFlag(conf *gcsConfig, flag string) []string {
 	var gcsoptions []string
 	path := fmt.Sprintf("gcs://%s/", path.Join(conf.bucket, conf.prefix))
+	if flag != "" && flag != defaultStorageFlag {
+		// now just set path to special flag
+		gcsoptions = append(gcsoptions, fmt.Sprintf("--%s=%s", flag, path))
+		return gcsoptions
+	}
 	gcsoptions = append(gcsoptions, fmt.Sprintf("--storage=%s", path))
+
 	if conf.storageClass != "" {
 		gcsoptions = append(gcsoptions, fmt.Sprintf("--gcs.storage-class=%s", conf.storageClass))
 	}
@@ -562,11 +579,17 @@ func newGcsStorageOption(conf *gcsConfig) []string {
 	return gcsoptions
 }
 
-// newAzblobStorageOption constructs the arg for --storage option and the remote path for br
-func newAzblobStorageOption(conf *azblobConfig) []string {
+// newAzblobStorageOption constructs the arg for --flag option and the remote path for br
+func newAzblobStorageOptionForFlag(conf *azblobConfig, flag string) []string {
 	var azblobOptions []string
 	path := fmt.Sprintf("azure://%s/", path.Join(conf.container, conf.prefix))
+	if flag != "" && flag != defaultStorageFlag {
+		// now just set path to special flag
+		azblobOptions = append(azblobOptions, fmt.Sprintf("--%s=%s", flag, path))
+		return azblobOptions
+	}
 	azblobOptions = append(azblobOptions, fmt.Sprintf("--storage=%s", path))
+
 	if conf.accessTier != "" {
 		azblobOptions = append(azblobOptions, fmt.Sprintf("--azblob.access-tier=%s", conf.accessTier))
 	}

cmd/backup-manager/app/util/util.go

@@ -155,7 +155,7 @@ func ConstructBRGlobalOptionsForBackup(backup *v1alpha1.Backup) ([]string, error
 		return nil, fmt.Errorf("no config for br in Backup %s/%s", backup.Namespace, backup.Name)
 	}
 	args = append(args, constructBRGlobalOptions(spec.BR)...)
-	storageArgs, err := genStorageArgs(backup.Spec.StorageProvider)
+	storageArgs, err := GenStorageArgsForFlag(backup.Spec.StorageProvider, "")
 	if err != nil {
 		return nil, err
 	}
@@ -224,7 +224,7 @@ func ConstructBRGlobalOptionsForRestore(restore *v1alpha1.Restore) ([]string, er
 		return nil, fmt.Errorf("no config for br in restore %s/%s", restore.Namespace, restore.Name)
 	}
 	args = append(args, constructBRGlobalOptions(config.BR)...)
-	storageArgs, err := genStorageArgs(restore.Spec.StorageProvider)
+	storageArgs, err := GenStorageArgsForFlag(restore.Spec.StorageProvider, "")
 	if err != nil {
 		return nil, err
 	}

docs/api-references/docs.md

@@ -1255,15 +1255,15 @@ StorageProvider
 </tr>
 <tr>
 <td>
-<code>logBackupStorageProvider</code></br>
+<code>pitrFullBackupStorageProvider</code></br>
 <em>
 <a href="#storageprovider">
 StorageProvider
 </a>
 </em>
 </td>
 <td>
-<p>LogBackupStorageProvider configures where and how log backup should be stored.</p>
+<p>PitrFullBackupStorageProvider configures where and how pitr dependent full backup should be stored.</p>
 </td>
 </tr>
 <tr>
@@ -13109,15 +13109,15 @@ StorageProvider
 </tr>
 <tr>
 <td>
-<code>logBackupStorageProvider</code></br>
+<code>pitrFullBackupStorageProvider</code></br>
 <em>
 <a href="#storageprovider">
 StorageProvider
 </a>
 </em>
 </td>
 <td>
-<p>LogBackupStorageProvider configures where and how log backup should be stored.</p>
+<p>PitrFullBackupStorageProvider configures where and how pitr dependent full backup should be stored.</p>
 </td>
 </tr>
 <tr>

manifests/crd.yaml

@@ -11928,7 +11928,9 @@ spec:
                 - volume
                 - volumeMount
                 type: object
-              logBackupStorageProvider:
+              logRestoreStartTs:
+                type: string
+              pitrFullBackupStorageProvider:
                 properties:
                   azblob:
                     properties:
@@ -12692,8 +12694,6 @@ spec:
                     - provider
                     type: object
                 type: object
-              logRestoreStartTs:
-                type: string
               pitrRestoredTs:
                 type: string
               podSecurityContext:

manifests/crd/v1/pingcap.com_restores.yaml

@@ -1151,7 +1151,9 @@ spec:
                 - volume
                 - volumeMount
                 type: object
-              logBackupStorageProvider:
+              logRestoreStartTs:
+                type: string
+              pitrFullBackupStorageProvider:
                 properties:
                   azblob:
                     properties:
@@ -1915,8 +1917,6 @@ spec:
                     - provider
                     type: object
                 type: object
-              logRestoreStartTs:
-                type: string
               pitrRestoredTs:
                 type: string
               podSecurityContext:

manifests/crd/v1beta1/pingcap.com_restores.yaml

@@ -1151,7 +1151,9 @@ spec:
               - volume
               - volumeMount
               type: object
-            logBackupStorageProvider:
+            logRestoreStartTs:
+              type: string
+            pitrFullBackupStorageProvider:
               properties:
                 azblob:
                   properties:
@@ -1915,8 +1917,6 @@ spec:
                   - provider
                   type: object
               type: object
-            logRestoreStartTs:
-              type: string
             pitrRestoredTs:
               type: string
             podSecurityContext:

manifests/crd_v1beta1.yaml

@@ -11920,7 +11920,9 @@ spec:
               - volume
               - volumeMount
               type: object
-            logBackupStorageProvider:
+            logRestoreStartTs:
+              type: string
+            pitrFullBackupStorageProvider:
               properties:
                 azblob:
                   properties:
@@ -12684,8 +12686,6 @@ spec:
                   - provider
                   type: object
               type: object
-            logRestoreStartTs:
-              type: string
             pitrRestoredTs:
               type: string
             podSecurityContext:

pkg/apis/pingcap/v1alpha1/types.go

@@ -2140,8 +2140,8 @@ type RestoreSpec struct {
 	TikvGCLifeTime *string `json:"tikvGCLifeTime,omitempty"`
 	// StorageProvider configures where and how backups should be stored.
 	StorageProvider `json:",inline"`
-	// LogBackupStorageProvider configures where and how log backup should be stored.
-	LogBackupStorageProvider StorageProvider `json:"logBackupStorageProvider,omitempty"`
+	// PitrFullBackupStorageProvider configures where and how pitr dependent full backup should be stored.
+	PitrFullBackupStorageProvider StorageProvider `json:"pitrFullBackupStorageProvider,omitempty"`
 	// The storageClassName of the persistent volume for Restore data storage.
 	// Defaults to Kubernetes default storage class.
 	// +optional

pkg/backup/restore/restore_manager.go

@@ -346,13 +346,19 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo
 		"restore",
 		fmt.Sprintf("--namespace=%s", ns),
 		fmt.Sprintf("--restoreName=%s", name),
+		fmt.Sprintf("--mode=%s", restore.Spec.Mode),
 	}
 	tikvImage := tc.TiKVImage()
 	_, tikvVersion := backuputil.ParseImage(tikvImage)
 	if tikvVersion != "" {
 		args = append(args, fmt.Sprintf("--tikvVersion=%s", tikvVersion))
 	}
 
+	// set pitr restore parameters
+	if restore.Spec.Mode == v1alpha1.RestoreModePiTR {
+		args = append(args, fmt.Sprintf("--pitrRestoredTs=%s", restore.Spec.PitrRestoredTs))
+	}
+
 	jobLabels := util.CombineStringMap(label.NewRestore().Instance(restore.GetInstanceName()).RestoreJob().Restore(name), restore.Labels)
 	podLabels := jobLabels
 	jobAnnotations := restore.Annotations