ebs br: make snapshot size calculation optional (#5202)

cmd/backup-manager/app/backup/manager.go

@@ -319,7 +319,7 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d
  defer func() {
  // Calculate the backup size for ebs backup job even if it fails
  if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize {
- fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider)
+ fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider, backup.Spec.CalcSizeLevel)
  if err != nil {
  klog.Errorf("Failed to calc volume snapshot backup, err: %v", err)
  return
@@ -332,6 +332,7 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d
  BackupSizeReadable: &backupSizeReadable,
  IncrementalBackupSize: &incrementalBackupSize,
  IncrementalBackupSizeReadable: &incrementalBackupSizeReadable,
+ TimeCompleted: &metav1.Time{Time: time.Now()},
  }
 
  if err := bm.StatusUpdater.Update(backup, nil, updateStatus); err != nil {

cmd/backup-manager/app/util/backup_size.go

@@ -45,16 +45,20 @@ import (
 // interface CalcVolSnapBackupSize called by backup and backup clean.
 
 const (
- // This value can be between 5 and 1,000; if MaxResults is given a value larger than 1,000, only 1,000 results are returned.
+ // DescribeSnapMaxReturnResult can be between 5 and 1,000; if MaxResults is given a value larger than 1,000, only 1,000 results are returned.
  DescribeSnapMaxReturnResult = 1000
- // This value can be between 100 and 1,0000, and charge ~0.6$/1 million request
+ // ListSnapMaxReturnResult can be between 100 and 10,000, and charge ~0.6$/1 million request
  ListSnapMaxReturnResult = 10000
- // This value can be between 1 and 50 due to aws service quota
+ // EbsApiConcurrency can be between 1 and 50 due to aws service quota
  EbsApiConcurrency = 40
+
+ CalculateFullSize = "full"
+ CalculateIncremental = "incremental"
+ CalculateAll = "all"
 )
 
 // CalcVolSnapBackupSize get snapshots from backup meta and then calc the backup size of snapshots.
-func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvider) (fullBackupSize int64, incrementalBackupSize int64, err error) {
+func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvider, level string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
  start := time.Now()
  // retrieves all snapshots from backup meta file
  volSnapshots, err := getSnapshotsFromBackupmeta(ctx, provider)
@@ -66,7 +70,7 @@ func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvide
  return 0, 0, err
  }
 
- fullBackupSize, incrementalBackupSize, err = calcBackupSize(ctx, volSnapshots)
+ fullBackupSize, incrementalBackupSize, err = calcBackupSize(ctx, volSnapshots, level)
 
  if err != nil {
  return 0, 0, err
@@ -136,7 +140,7 @@ func getSnapshotsFromBackupmeta(ctx context.Context, provider v1alpha1.StoragePr
 }
 
 // calcBackupSize get a volume-snapshots backup size
-func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
+func calcBackupSize(ctx context.Context, volumes map[string]string, level string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
  var apiReqCount, incrementalApiReqCount uint64
 
  workerPool := util.NewWorkerPool(EbsApiConcurrency, "list snapshot size")
@@ -147,30 +151,35 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS
  volumeId := vid
  // sort snapshots by timestamp
  workerPool.ApplyOnErrorGroup(eg, func() error {
- snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId)
- if err != nil {
- return err
+ var snapSize uint64
+ if level == CalculateAll || level == CalculateFullSize {
+ snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId)
+ if err != nil {
+ return err
+ }
+ atomic.AddInt64(&fullBackupSize, int64(snapSize))
+ atomic.AddUint64(&apiReqCount, apiReq)
  }
- atomic.AddInt64(&fullBackupSize, int64(snapSize))
- atomic.AddUint64(&apiReqCount, apiReq)
 
- volSnapshots, err := getVolSnapshots(volumeId)
- if err != nil {
- return err
- }
- prevSnapshotId, existed := getPrevSnapshotId(snapshotId, volSnapshots)
- if !existed {
- // if there is no previous snapshot, means it's the first snapshot, uses its full size as incremental size
- atomic.AddInt64(&incrementalBackupSize, int64(snapSize))
- return nil
+ if level == CalculateAll || level == CalculateIncremental {
+ volSnapshots, err := getVolSnapshots(volumeId)
+ if err != nil {
+ return err
+ }
+ prevSnapshotId, existed := getPrevSnapshotId(snapshotId, volSnapshots)
+ if !existed {
+ // if there is no previous snapshot, means it's the first snapshot, uses its full size as incremental size
+ atomic.AddInt64(&incrementalBackupSize, int64(snapSize))
+ return nil
+ }
+ klog.Infof("get previous snapshot %s of snapshot %s, volume %s", prevSnapshotId, snapshotId, volumeId)
+ incrementalSnapSize, incrementalApiReq, err := calculateChangedBlocksSize(volumeId, prevSnapshotId, snapshotId)
+ if err != nil {
+ return err
+ }
+ atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize))
+ atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq)
  }
- klog.Infof("get previous snapshot %s of snapshot %s, volume %s", prevSnapshotId, snapshotId, volumeId)
- incrementalSnapSize, incrementalApiReq, err := calculateChangedBlocksSize(volumeId, prevSnapshotId, snapshotId)
- if err != nil {
- return err
- }
- atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize))
- atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq)
  return nil
  })
  }
@@ -217,8 +226,10 @@ func calculateSnapshotSize(volumeId, snapshotId string) (uint64, uint64, error)
  }
  nextToken = resp.NextToken
  }
+
  klog.Infof("full snapshot size %s, num of ListSnapshotBlocks request %d, snapshot id %s, volume id %s",
  humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId)
+
  return snapshotSize, numApiReq, nil
 }
 
@@ -230,6 +241,7 @@ func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uin
 
  klog.Infof("start to calculate incremental snapshot size for %s, base on prev snapshot %s, volume id %s",
  snapshotId, preSnapshotId, volumeId)
+
  ebsSession, err := util.NewEBSSession(util.CloudAPIConcurrency)
  if err != nil {
  klog.Errorf("new a ebs session failure.")
@@ -264,6 +276,7 @@ func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uin
  }
  nextToken = resp.NextToken
  }
+
  klog.Infof("incremental snapshot size %s, num of api ListChangedBlocks request %d, snapshot id %s, volume id %s",
  humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId)
  return snapshotSize, numApiReq, nil

docs/api-references/docs.md

@@ -270,6 +270,18 @@ bool
 </tr>
 <tr>
 <td>
+<code>calcSizeLevel</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>federalVolumeBackupPhase</code></br>
 <em>
 <a href="#federalvolumebackupphase">
@@ -4036,6 +4048,18 @@ bool
 </tr>
 <tr>
 <td>
+<code>calcSizeLevel</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>federalVolumeBackupPhase</code></br>
 <em>
 <a href="#federalvolumebackupphase">

docs/api-references/federation-docs.md

@@ -695,6 +695,18 @@ string
 <p>PriorityClassName of Backup Job Pods</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>calcSizeLevel</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="volumebackupmemberstatus">VolumeBackupMemberStatus</h3>

manifests/crd.yaml

@@ -503,6 +503,9 @@ spec:
  required:
  - cluster
  type: object
+ calcSizeLevel:
+ default: all
+ type: string
  cleanOption:
  properties:
  backoffEnabled:
@@ -2126,6 +2129,9 @@ spec:
  required:
  - cluster
  type: object
+ calcSizeLevel:
+ default: all
+ type: string
  cleanOption:
  properties:
  backoffEnabled:
@@ -3553,6 +3559,9 @@ spec:
  required:
  - cluster
  type: object
+ calcSizeLevel:
+ default: all
+ type: string
  cleanOption:
  properties:
  backoffEnabled:

manifests/crd/federation/v1/federation.pingcap.com_volumebackups.yaml

@@ -90,6 +90,9 @@ spec:
  sendCredToTikv:
  type: boolean
  type: object
+ calcSizeLevel:
+ default: all
+ type: string
  cleanPolicy:
  type: string
  env:

manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml

@@ -98,6 +98,9 @@ spec:
  sendCredToTikv:
  type: boolean
  type: object
+ calcSizeLevel:
+ default: all
+ type: string
  cleanPolicy:
  type: string
  env:

manifests/crd/v1/pingcap.com_backups.yaml

@@ -503,6 +503,9 @@ spec:
  required:
  - cluster
  type: object
+ calcSizeLevel:
+ default: all
+ type: string
  cleanOption:
  properties:
  backoffEnabled:

manifests/crd/v1/pingcap.com_backupschedules.yaml

@@ -478,6 +478,9 @@ spec:
  required:
  - cluster
  type: object
+ calcSizeLevel:
+ default: all
+ type: string
  cleanOption:
  properties:
  backoffEnabled:
@@ -1905,6 +1908,9 @@ spec:
  required:
  - cluster
  type: object
+ calcSizeLevel:
+ default: all
+ type: string
  cleanOption:
  properties:
  backoffEnabled:

manifests/federation-crd.yaml

@@ -90,6 +90,9 @@ spec:
  sendCredToTikv:
  type: boolean
  type: object
+ calcSizeLevel:
+ default: all
+ type: string
  cleanPolicy:
  type: string
  env:
@@ -1143,6 +1146,9 @@ spec:
  sendCredToTikv:
  type: boolean
  type: object
+ calcSizeLevel:
+ default: all
+ type: string
  cleanPolicy:
  type: string
  env:

pkg/apis/federation/pingcap/v1alpha1/types.go

@@ -113,6 +113,10 @@ type VolumeBackupMemberSpec struct {
  CleanPolicy pingcapv1alpha1.CleanPolicyType `json:"cleanPolicy,omitempty"`
  // PriorityClassName of Backup Job Pods
  PriorityClassName string `json:"priorityClassName,omitempty"`
+ // CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup
+ // +optional
+ // +kubebuilder:default="all"
+ CalcSizeLevel string `json:"calcSizeLevel,omitempty"`
 }
 
 // BRConfig contains config for BR

pkg/apis/pingcap/v1alpha1/types.go

@@ -1949,6 +1949,10 @@ type BackupSpec struct {
  // LogStop indicates that will stop the log backup.
  // +optional
  LogStop bool `json:"logStop,omitempty"`
+ // CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup
+ // +optional
+ // +kubebuilder:default="all"
+ CalcSizeLevel string `json:"calcSizeLevel,omitempty"`
  // FederalVolumeBackupPhase indicates which phase to execute in federal volume backup
  // +optional
  FederalVolumeBackupPhase FederalVolumeBackupPhase `json:"federalVolumeBackupPhase,omitempty"`

pkg/backup/snapshotter/snapshotter_test.go

@@ -429,8 +429,9 @@ func TestGenerateBackupMetadata(t *testing.T) {
  Annotations: make(map[string]string),
  },
  Spec: v1alpha1.BackupSpec{
- Type: v1alpha1.BackupTypeFull,
- Mode: v1alpha1.BackupModeVolumeSnapshot,
+ Type: v1alpha1.BackupTypeFull,
+ Mode: v1alpha1.BackupModeVolumeSnapshot,
+ CalcSizeLevel: "all",
  },
  },
  wantSSNil: false,