*: address conflict

Signed-off-by: BornChanger <dawn_catcher@126.com>
pingcap · Jul 28, 2023 · 6e873eb · 6e873eb
1 parent b098c54
commit 6e873eb
Show file tree

Hide file tree

Showing 15 changed files with 101 additions and 66 deletions.
diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go
@@ -318,17 +318,8 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d
 
 	defer func() {
 		// Calculate the backup size for ebs backup job even if it fails
-<<<<<<< HEAD
-<<<<<<< HEAD
 		if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize {
-			fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider)
-=======
-		if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize && !bm.Initialize {
-=======
-		if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize {
->>>>>>> f91a7b4e7 (*: fix typo)
 			fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider, backup.Spec.CalcSizeLevel)
->>>>>>> 44eefb26c (*: code format)
 			if err != nil {
 				klog.Errorf("Failed to calc volume snapshot backup, err: %v", err)
 				return

diff --git a/cmd/backup-manager/app/util/backup_size.go b/cmd/backup-manager/app/util/backup_size.go
@@ -27,7 +27,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ebs"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/dustin/go-humanize"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
@@ -47,11 +46,11 @@ import (
 // interface CalcVolSnapBackupSize called by backup and backup clean.
 
 const (
-	// This value can be between 5 and 1,000; if MaxResults is given a value larger than 1,000, only 1,000 results are returned.
+	// DescribeSnapMaxReturnResult can be between 5 and 1,000; if MaxResults is given a value larger than 1,000, only 1,000 results are returned.
 	DescribeSnapMaxReturnResult = 1000
-	// This value can be between 100 and 1,0000, and charge ~0.6$/1 million request
+	// ListSnapMaxReturnResult can be between 100 and 10,000, and charge ~0.6$/1 million request
 	ListSnapMaxReturnResult = 10000
-	// This value can be between 1 and 50 due to aws service quota
+	// EbsApiConcurrency can be between 1 and 50 due to aws service quota
 	EbsApiConcurrency = 40
 
 	CalculateFullSize    = "full"
@@ -60,7 +59,7 @@ const (
 )
 
 // CalcVolSnapBackupSize get snapshots from backup meta and then calc the backup size of snapshots.
-func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvider) (fullBackupSize int64, incrementalBackupSize int64, err error) {
+func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvider, level string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
 	start := time.Now()
 	// retrieves all snapshots from backup meta file
 	volSnapshots, err := getSnapshotsFromBackupmeta(ctx, provider)
@@ -72,7 +71,7 @@ func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvide
 		return 0, 0, err
 	}
 
-	fullBackupSize, incrementalBackupSize, err = calcBackupSize(ctx, volSnapshots)
+	fullBackupSize, incrementalBackupSize, err = calcBackupSize(ctx, volSnapshots, level)
 
 	if err != nil {
 		return 0, 0, err
@@ -142,7 +141,7 @@ func getSnapshotsFromBackupmeta(ctx context.Context, provider v1alpha1.StoragePr
 }
 
 // calcBackupSize get a volume-snapshots backup size
-func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
+func calcBackupSize(ctx context.Context, volumes map[string]string, level string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
 	var apiReqCount, incrementalApiReqCount uint64
 
 	workerPool := util.NewWorkerPool(EbsApiConcurrency, "list snapshot size")
@@ -153,11 +152,6 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS
 		volumeId := vid
 		// sort snapshots by timestamp
 		workerPool.ApplyOnErrorGroup(eg, func() error {
-<<<<<<< HEAD
-			snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId)
-			if err != nil {
-				return err
-=======
 			var snapSize uint64
 			if level == CalculateAll || level == CalculateFullSize {
 				snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId)
@@ -166,22 +160,9 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS
 				}
 				atomic.AddInt64(&fullBackupSize, int64(snapSize))
 				atomic.AddUint64(&apiReqCount, apiReq)
->>>>>>> 33580d9dc (*: manual cp)
 			}
-<<<<<<< HEAD
-			atomic.AddInt64(&fullBackupSize, int64(snapSize))
-			atomic.AddUint64(&apiReqCount, apiReq)
-
-<<<<<<< HEAD
-			volSnapshots, err := getVolSnapshots(volumeId)
-			if err != nil {
-				return err
-=======
-			if level == backup.CalculateAll || level == backup.CalculateIncremental {
-=======
 
 			if level == CalculateAll || level == CalculateIncremental {
->>>>>>> 44eefb26c (*: code format)
 				volSnapshots, err := getVolSnapshots(volumeId)
 				if err != nil {
 					return err
@@ -199,21 +180,7 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS
 				}
 				atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize))
 				atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq)
->>>>>>> 33580d9dc (*: manual cp)
-			}
-			prevSnapshotId, existed := getPrevSnapshotId(snapshotId, volSnapshots)
-			if !existed {
-				// if there is no previous snapshot, means it's the first snapshot, uses its full size as incremental size
-				atomic.AddInt64(&incrementalBackupSize, int64(snapSize))
-				return nil
 			}
-			klog.Infof("get previous snapshot %s of snapshot %s, volume %s", prevSnapshotId, snapshotId, volumeId)
-			incrementalSnapSize, incrementalApiReq, err := calculateChangedBlocksSize(volumeId, prevSnapshotId, snapshotId)
-			if err != nil {
-				return err
-			}
-			atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize))
-			atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq)
 			return nil
 		})
 	}
@@ -260,31 +227,22 @@ func calculateSnapshotSize(volumeId, snapshotId string) (uint64, uint64, error)
 		}
 		nextToken = resp.NextToken
 	}
-<<<<<<< HEAD
+
 	klog.Infof("full snapshot size %s, num of ListSnapshotBlocks request %d, snapshot id %s, volume id %s",
 		humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId)
-=======
-	klog.Infof("full backup snapshot size %d bytes, num of ListSnapshotBlocks request %d, snapshot id %s, volume id %s", humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId)
->>>>>>> 33580d9dc (*: manual cp)
+
 	return snapshotSize, numApiReq, nil
 }
 
 // calculateChangedBlocksSize calculates changed blocks total size in bytes between two snapshots with common ancestry.
-<<<<<<< HEAD
 func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uint64, uint64, error) {
-=======
-func calculateChangedBlocksSize(volumeId, preSnapshotId string, snapshotId string) (uint64, uint64, error) {
->>>>>>> 33580d9dc (*: manual cp)
 	var numBlocks int
 	var snapshotSize uint64
 	var numApiReq uint64
 
-<<<<<<< HEAD
 	klog.Infof("start to calculate incremental snapshot size for %s, base on prev snapshot %s, volume id %s",
 		snapshotId, preSnapshotId, volumeId)
-=======
-	klog.Infof("the calc snapshot size for %s, base on prev snapshot %s, volume id %s", snapshotId, preSnapshotId, volumeId)
->>>>>>> 33580d9dc (*: manual cp)
+
 	ebsSession, err := util.NewEBSSession(util.CloudAPIConcurrency)
 	if err != nil {
 		klog.Errorf("new a ebs session failure.")
@@ -319,12 +277,9 @@ func calculateChangedBlocksSize(volumeId, preSnapshotId string, snapshotId strin
 		}
 		nextToken = resp.NextToken
 	}
-<<<<<<< HEAD
+
 	klog.Infof("incremental snapshot size %s, num of api ListChangedBlocks request %d, snapshot id %s, volume id %s",
 		humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId)
-=======
-	klog.Infof("the total size of snapshot %d, num of api ListChangedBlocks request %d, snapshot id %s, volume id %s", humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId)
->>>>>>> 33580d9dc (*: manual cp)
 	return snapshotSize, numApiReq, nil
 }
 

diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md
@@ -270,6 +270,18 @@ bool
 </tr>
 <tr>
 <td>
+<code>calcSizeLevel</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>federalVolumeBackupPhase</code></br>
 <em>
 <a href="#federalvolumebackupphase">
@@ -4036,6 +4048,18 @@ bool
 </tr>
 <tr>
 <td>
+<code>calcSizeLevel</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>federalVolumeBackupPhase</code></br>
 <em>
 <a href="#federalvolumebackupphase">

diff --git a/docs/api-references/federation-docs.md b/docs/api-references/federation-docs.md
@@ -695,6 +695,18 @@ string
 <p>PriorityClassName of Backup Job Pods</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>calcSizeLevel</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="volumebackupmemberstatus">VolumeBackupMemberStatus</h3>

diff --git a/manifests/crd.yaml b/manifests/crd.yaml
@@ -503,6 +503,9 @@ spec:
                 required:
                 - cluster
                 type: object
+              calcSizeLevel:
+                default: all
+                type: string
               cleanOption:
                 properties:
                   backoffEnabled:
@@ -2126,6 +2129,9 @@ spec:
                     required:
                     - cluster
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanOption:
                     properties:
                       backoffEnabled:
@@ -3553,6 +3559,9 @@ spec:
                     required:
                     - cluster
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanOption:
                     properties:
                       backoffEnabled:

diff --git a/manifests/crd/federation/v1/federation.pingcap.com_volumebackups.yaml b/manifests/crd/federation/v1/federation.pingcap.com_volumebackups.yaml
@@ -90,6 +90,9 @@ spec:
                       sendCredToTikv:
                         type: boolean
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanPolicy:
                     type: string
                   env:

diff --git a/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml b/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml
@@ -98,6 +98,9 @@ spec:
                           sendCredToTikv:
                             type: boolean
                         type: object
+                      calcSizeLevel:
+                        default: all
+                        type: string
                       cleanPolicy:
                         type: string
                       env:

diff --git a/manifests/crd/v1/pingcap.com_backups.yaml b/manifests/crd/v1/pingcap.com_backups.yaml
@@ -503,6 +503,9 @@ spec:
                 required:
                 - cluster
                 type: object
+              calcSizeLevel:
+                default: all
+                type: string
               cleanOption:
                 properties:
                   backoffEnabled:

diff --git a/manifests/crd/v1/pingcap.com_backupschedules.yaml b/manifests/crd/v1/pingcap.com_backupschedules.yaml
@@ -478,6 +478,9 @@ spec:
                     required:
                     - cluster
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanOption:
                     properties:
                       backoffEnabled:
@@ -1905,6 +1908,9 @@ spec:
                     required:
                     - cluster
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanOption:
                     properties:
                       backoffEnabled:

diff --git a/manifests/federation-crd.yaml b/manifests/federation-crd.yaml
@@ -90,6 +90,9 @@ spec:
                       sendCredToTikv:
                         type: boolean
                     type: object
+                  calcSizeLevel:
+                    default: all
+                    type: string
                   cleanPolicy:
                     type: string
                   env:
@@ -1143,6 +1146,9 @@ spec:
                           sendCredToTikv:
                             type: boolean
                         type: object
+                      calcSizeLevel:
+                        default: all
+                        type: string
                       cleanPolicy:
                         type: string
                       env:

diff --git a/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go
diff --git a/pkg/apis/federation/pingcap/v1alpha1/types.go b/pkg/apis/federation/pingcap/v1alpha1/types.go
@@ -113,6 +113,10 @@ type VolumeBackupMemberSpec struct {
 	CleanPolicy pingcapv1alpha1.CleanPolicyType `json:"cleanPolicy,omitempty"`
 	// PriorityClassName of Backup Job Pods
 	PriorityClassName string `json:"priorityClassName,omitempty"`
+	// CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup
+	// +optional
+	// +kubebuilder:default="all"
+	CalcSizeLevel string `json:"calcSizeLevel,omitempty"`
 }
 
 // BRConfig contains config for BR

diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go
diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go
@@ -1949,6 +1949,10 @@ type BackupSpec struct {
 	// LogStop indicates that will stop the log backup.
 	// +optional
 	LogStop bool `json:"logStop,omitempty"`
+	// CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup
+	// +optional
+	// +kubebuilder:default="all"
+	CalcSizeLevel string `json:"calcSizeLevel,omitempty"`
 	// FederalVolumeBackupPhase indicates which phase to execute in federal volume backup
 	// +optional
 	FederalVolumeBackupPhase FederalVolumeBackupPhase `json:"federalVolumeBackupPhase,omitempty"`

diff --git a/pkg/backup/snapshotter/snapshotter_test.go b/pkg/backup/snapshotter/snapshotter_test.go
@@ -429,8 +429,9 @@ func TestGenerateBackupMetadata(t *testing.T) {
 					Annotations: make(map[string]string),
 				},
 				Spec: v1alpha1.BackupSpec{
-					Type: v1alpha1.BackupTypeFull,
-					Mode: v1alpha1.BackupModeVolumeSnapshot,
+					Type:          v1alpha1.BackupTypeFull,
+					Mode:          v1alpha1.BackupModeVolumeSnapshot,
+					CalcSizeLevel: "all",
 				},
 			},
 			wantSSNil: false,