Skip to content

Commit

Permalink
ebs br: make snapshot size calculation optional (#5202) (#5214)
Browse files Browse the repository at this point in the history 
Signed-off-by: BornChanger <dawn_catcher@126.com>
Co-authored-by: BornChanger <dawn_catcher@126.com>
  • Loading branch information
@ti-chi-bot @BornChanger
ti-chi-bot and BornChanger authored Jul 31, 2023
1 parent 9738c4f commit d6bd886
Show file tree
Hide file tree
Showing 21 changed files with 153 additions and 30 deletions.
3 changes: 2 additions & 1 deletion cmd/backup-manager/app/backup/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d
defer func() {
// Calculate the backup size for ebs backup job even if it fails
if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize {
fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider)
fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider, backup.Spec.CalcSizeLevel)
if err != nil {
klog.Errorf("Failed to calc volume snapshot backup, err: %v", err)
return
Expand All @@ -332,6 +332,7 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d
BackupSizeReadable: &backupSizeReadable,
IncrementalBackupSize: &incrementalBackupSize,
IncrementalBackupSizeReadable: &incrementalBackupSizeReadable,
TimeCompleted: &metav1.Time{Time: time.Now()},
}

if err := bm.StatusUpdater.Update(backup, nil, updateStatus); err != nil {
Expand Down
67 changes: 40 additions & 27 deletions cmd/backup-manager/app/util/backup_size.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,16 +45,20 @@ import (
// interface CalcVolSnapBackupSize called by backup and backup clean.

const (
// This value can be between 5 and 1,000; if MaxResults is given a value larger than 1,000, only 1,000 results are returned.
// DescribeSnapMaxReturnResult can be between 5 and 1,000; if MaxResults is given a value larger than 1,000, only 1,000 results are returned.
DescribeSnapMaxReturnResult = 1000
// This value can be between 100 and 1,0000, and charge ~0.6$/1 million request
// ListSnapMaxReturnResult can be between 100 and 10,000, and charge ~0.6$/1 million request
ListSnapMaxReturnResult = 10000
// This value can be between 1 and 50 due to aws service quota
// EbsApiConcurrency can be between 1 and 50 due to aws service quota
EbsApiConcurrency = 40

CalculateFullSize = "full"
CalculateIncremental = "incremental"
CalculateAll = "all"
)

// CalcVolSnapBackupSize get snapshots from backup meta and then calc the backup size of snapshots.
func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvider) (fullBackupSize int64, incrementalBackupSize int64, err error) {
func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvider, level string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
start := time.Now()
// retrieves all snapshots from backup meta file
volSnapshots, err := getSnapshotsFromBackupmeta(ctx, provider)
Expand All @@ -66,7 +70,7 @@ func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvide
return 0, 0, err
}

fullBackupSize, incrementalBackupSize, err = calcBackupSize(ctx, volSnapshots)
fullBackupSize, incrementalBackupSize, err = calcBackupSize(ctx, volSnapshots, level)

if err != nil {
return 0, 0, err
Expand Down Expand Up @@ -136,7 +140,7 @@ func getSnapshotsFromBackupmeta(ctx context.Context, provider v1alpha1.StoragePr
}

// calcBackupSize get a volume-snapshots backup size
func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
func calcBackupSize(ctx context.Context, volumes map[string]string, level string) (fullBackupSize int64, incrementalBackupSize int64, err error) {
var apiReqCount, incrementalApiReqCount uint64

workerPool := util.NewWorkerPool(EbsApiConcurrency, "list snapshot size")
Expand All @@ -147,30 +151,35 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS
volumeId := vid
// sort snapshots by timestamp
workerPool.ApplyOnErrorGroup(eg, func() error {
snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId)
if err != nil {
return err
var snapSize uint64
if level == CalculateAll || level == CalculateFullSize {
snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId)
if err != nil {
return err
}
atomic.AddInt64(&fullBackupSize, int64(snapSize))
atomic.AddUint64(&apiReqCount, apiReq)
}
atomic.AddInt64(&fullBackupSize, int64(snapSize))
atomic.AddUint64(&apiReqCount, apiReq)

volSnapshots, err := getVolSnapshots(volumeId)
if err != nil {
return err
}
prevSnapshotId, existed := getPrevSnapshotId(snapshotId, volSnapshots)
if !existed {
// if there is no previous snapshot, means it's the first snapshot, uses its full size as incremental size
atomic.AddInt64(&incrementalBackupSize, int64(snapSize))
return nil
if level == CalculateAll || level == CalculateIncremental {
volSnapshots, err := getVolSnapshots(volumeId)
if err != nil {
return err
}
prevSnapshotId, existed := getPrevSnapshotId(snapshotId, volSnapshots)
if !existed {
// if there is no previous snapshot, means it's the first snapshot, uses its full size as incremental size
atomic.AddInt64(&incrementalBackupSize, int64(snapSize))
return nil
}
klog.Infof("get previous snapshot %s of snapshot %s, volume %s", prevSnapshotId, snapshotId, volumeId)
incrementalSnapSize, incrementalApiReq, err := calculateChangedBlocksSize(volumeId, prevSnapshotId, snapshotId)
if err != nil {
return err
}
atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize))
atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq)
}
klog.Infof("get previous snapshot %s of snapshot %s, volume %s", prevSnapshotId, snapshotId, volumeId)
incrementalSnapSize, incrementalApiReq, err := calculateChangedBlocksSize(volumeId, prevSnapshotId, snapshotId)
if err != nil {
return err
}
atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize))
atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq)
return nil
})
}
Expand Down Expand Up @@ -217,8 +226,10 @@ func calculateSnapshotSize(volumeId, snapshotId string) (uint64, uint64, error)
}
nextToken = resp.NextToken
}

klog.Infof("full snapshot size %s, num of ListSnapshotBlocks request %d, snapshot id %s, volume id %s",
humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId)

return snapshotSize, numApiReq, nil
}

Expand All @@ -230,6 +241,7 @@ func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uin

klog.Infof("start to calculate incremental snapshot size for %s, base on prev snapshot %s, volume id %s",
snapshotId, preSnapshotId, volumeId)

ebsSession, err := util.NewEBSSession(util.CloudAPIConcurrency)
if err != nil {
klog.Errorf("new a ebs session failure.")
Expand Down Expand Up @@ -264,6 +276,7 @@ func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uin
}
nextToken = resp.NextToken
}

klog.Infof("incremental snapshot size %s, num of api ListChangedBlocks request %d, snapshot id %s, volume id %s",
humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId)
return snapshotSize, numApiReq, nil
Expand Down
24 changes: 24 additions & 0 deletions docs/api-references/docs.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,18 @@ bool
</tr>
<tr>
<td>
<code>calcSizeLevel</code></br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
</td>
</tr>
<tr>
<td>
<code>federalVolumeBackupPhase</code></br>
<em>
<a href="#federalvolumebackupphase">
Expand Down Expand Up @@ -4036,6 +4048,18 @@ bool
</tr>
<tr>
<td>
<code>calcSizeLevel</code></br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
</td>
</tr>
<tr>
<td>
<code>federalVolumeBackupPhase</code></br>
<em>
<a href="#federalvolumebackupphase">
Expand Down
12 changes: 12 additions & 0 deletions docs/api-references/federation-docs.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -695,6 +695,18 @@ string
<p>PriorityClassName of Backup Job Pods</p>
</td>
</tr>
<tr>
<td>
<code>calcSizeLevel</code></br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup</p>
</td>
</tr>
</tbody>
</table>
<h3 id="volumebackupmemberstatus">VolumeBackupMemberStatus</h3>
Expand Down
9 changes: 9 additions & 0 deletions manifests/crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,9 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
default: all
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down Expand Up @@ -1930,6 +1933,9 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
default: all
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down Expand Up @@ -3253,6 +3259,9 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
default: all
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down
3 changes: 3 additions & 0 deletions manifests/crd/federation/v1/federation.pingcap.com_volumebackups.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ spec:
sendCredToTikv:
type: boolean
type: object
calcSizeLevel:
default: all
type: string
cleanPolicy:
type: string
env:
Expand Down
3 changes: 3 additions & 0 deletions manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ spec:
sendCredToTikv:
type: boolean
type: object
calcSizeLevel:
default: all
type: string
cleanPolicy:
type: string
env:
Expand Down
2 changes: 2 additions & 0 deletions manifests/crd/federation/v1beta1/federation.pingcap.com_volumebackups.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ spec:
sendCredToTikv:
type: boolean
type: object
calcSizeLevel:
type: string
cleanPolicy:
type: string
env:
Expand Down
2 changes: 2 additions & 0 deletions manifests/crd/federation/v1beta1/federation.pingcap.com_volumebackupschedules.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ spec:
sendCredToTikv:
type: boolean
type: object
calcSizeLevel:
type: string
cleanPolicy:
type: string
env:
Expand Down
3 changes: 3 additions & 0 deletions manifests/crd/v1/pingcap.com_backups.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,9 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
default: all
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down
6 changes: 6 additions & 0 deletions manifests/crd/v1/pingcap.com_backupschedules.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,9 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
default: all
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down Expand Up @@ -1709,6 +1712,9 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
default: all
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down
2 changes: 2 additions & 0 deletions manifests/crd/v1beta1/pingcap.com_backups.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,8 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down
4 changes: 4 additions & 0 deletions manifests/crd/v1beta1/pingcap.com_backupschedules.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,8 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down Expand Up @@ -1700,6 +1702,8 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down
6 changes: 6 additions & 0 deletions manifests/crd_v1beta1.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,8 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down Expand Up @@ -1923,6 +1925,8 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down Expand Up @@ -3241,6 +3245,8 @@ spec:
required:
- cluster
type: object
calcSizeLevel:
type: string
cleanOption:
properties:
backoffEnabled:
Expand Down
6 changes: 6 additions & 0 deletions manifests/federation-crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ spec:
sendCredToTikv:
type: boolean
type: object
calcSizeLevel:
default: all
type: string
cleanPolicy:
type: string
env:
Expand Down Expand Up @@ -1133,6 +1136,9 @@ spec:
sendCredToTikv:
type: boolean
type: object
calcSizeLevel:
default: all
type: string
cleanPolicy:
type: string
env:
Expand Down
4 changes: 4 additions & 0 deletions manifests/federation-crd_v1beta1.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ spec:
sendCredToTikv:
type: boolean
type: object
calcSizeLevel:
type: string
cleanPolicy:
type: string
env:
Expand Down Expand Up @@ -1135,6 +1137,8 @@ spec:
sendCredToTikv:
type: boolean
type: object
calcSizeLevel:
type: string
cleanPolicy:
type: string
env:
Expand Down
Loading

0 comments on commit d6bd886

Please sign in to comment.