From 754f51f7a2259cf578a595a8a7ec31313df64ee7 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Thu, 27 Jul 2023 23:49:08 +0800 Subject: [PATCH 1/7] *: manual cp Signed-off-by: BornChanger --- cmd/backup-manager/app/util/backup_size.go | 49 ++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/cmd/backup-manager/app/util/backup_size.go b/cmd/backup-manager/app/util/backup_size.go index 18c47d9871..b1bc864732 100644 --- a/cmd/backup-manager/app/util/backup_size.go +++ b/cmd/backup-manager/app/util/backup_size.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/dustin/go-humanize" "sort" "strings" "sync/atomic" @@ -147,16 +148,48 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS volumeId := vid // sort snapshots by timestamp workerPool.ApplyOnErrorGroup(eg, func() error { +<<<<<<< HEAD snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId) if err != nil { return err +======= + var snapSize uint64 + if level == backup.CalculateFullSize || level == backup.CalculateFullSize { + snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId) + if err != nil { + return err + } + atomic.AddInt64(&fullBackupSize, int64(snapSize)) + atomic.AddUint64(&apiReqCount, apiReq) +>>>>>>> 33580d9dc (*: manual cp) } atomic.AddInt64(&fullBackupSize, int64(snapSize)) atomic.AddUint64(&apiReqCount, apiReq) +<<<<<<< HEAD volSnapshots, err := getVolSnapshots(volumeId) if err != nil { return err +======= + if level == backup.CalculateAll || level == backup.CalculateIncremental { + volSnapshots, err := getVolSnapshots(volumeId) + if err != nil { + return err + } + prevSnapshotId, existed := getPrevSnapshotId(snapshotId, volSnapshots) + if !existed { + // if there is no previous snapshot, means it's the first snapshot, uses its full size as incremental size + atomic.AddInt64(&incrementalBackupSize, int64(snapSize)) + return nil + } + klog.Infof("get previous snapshot %s of snapshot %s, volume %s", prevSnapshotId, snapshotId, volumeId) + incrementalSnapSize, incrementalApiReq, err := calculateChangedBlocksSize(volumeId, prevSnapshotId, snapshotId) + if err != nil { + return err + } + atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize)) + atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq) +>>>>>>> 33580d9dc (*: manual cp) } prevSnapshotId, existed := getPrevSnapshotId(snapshotId, volSnapshots) if !existed { @@ -217,19 +250,31 @@ func calculateSnapshotSize(volumeId, snapshotId string) (uint64, uint64, error) } nextToken = resp.NextToken } +<<<<<<< HEAD klog.Infof("full snapshot size %s, num of ListSnapshotBlocks request %d, snapshot id %s, volume id %s", humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId) +======= + klog.Infof("full backup snapshot size %d bytes, num of ListSnapshotBlocks request %d, snapshot id %s, volume id %s", humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId) +>>>>>>> 33580d9dc (*: manual cp) return snapshotSize, numApiReq, nil } // calculateChangedBlocksSize calculates changed blocks total size in bytes between two snapshots with common ancestry. +<<<<<<< HEAD func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uint64, uint64, error) { +======= +func calculateChangedBlocksSize(volumeId, preSnapshotId string, snapshotId string) (uint64, uint64, error) { +>>>>>>> 33580d9dc (*: manual cp) var numBlocks int var snapshotSize uint64 var numApiReq uint64 +<<<<<<< HEAD klog.Infof("start to calculate incremental snapshot size for %s, base on prev snapshot %s, volume id %s", snapshotId, preSnapshotId, volumeId) +======= + klog.Infof("the calc snapshot size for %s, base on prev snapshot %s, volume id %s", snapshotId, preSnapshotId, volumeId) +>>>>>>> 33580d9dc (*: manual cp) ebsSession, err := util.NewEBSSession(util.CloudAPIConcurrency) if err != nil { klog.Errorf("new a ebs session failure.") @@ -264,8 +309,12 @@ func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uin } nextToken = resp.NextToken } +<<<<<<< HEAD klog.Infof("incremental snapshot size %s, num of api ListChangedBlocks request %d, snapshot id %s, volume id %s", humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId) +======= + klog.Infof("the total size of snapshot %d, num of api ListChangedBlocks request %d, snapshot id %s, volume id %s", humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId) +>>>>>>> 33580d9dc (*: manual cp) return snapshotSize, numApiReq, nil } From 219336918a62427e65084d44592ff64c87497f33 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Thu, 27 Jul 2023 23:54:02 +0800 Subject: [PATCH 2/7] *: code format Signed-off-by: BornChanger --- cmd/backup-manager/app/util/backup_size.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/backup-manager/app/util/backup_size.go b/cmd/backup-manager/app/util/backup_size.go index b1bc864732..5f14cdae8f 100644 --- a/cmd/backup-manager/app/util/backup_size.go +++ b/cmd/backup-manager/app/util/backup_size.go @@ -17,12 +17,13 @@ import ( "context" "encoding/json" "fmt" - "github.com/dustin/go-humanize" "sort" "strings" "sync/atomic" "time" + "github.com/dustin/go-humanize" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ebs" "github.com/aws/aws-sdk-go/service/ec2" From e3ca73a7a7f2997fd39e3d1063280ee7d38679cb Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 28 Jul 2023 00:01:15 +0800 Subject: [PATCH 3/7] *: code format Signed-off-by: BornChanger --- cmd/backup-manager/app/backup/manager.go | 5 +++++ cmd/backup-manager/app/util/backup_size.go | 11 ++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index 2425c322a2..8dedbd0a35 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -318,8 +318,13 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d defer func() { // Calculate the backup size for ebs backup job even if it fails +<<<<<<< HEAD if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize { fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider) +======= + if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize && !bm.Initialize { + fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider, backup.Spec.CalcSizeLevel) +>>>>>>> 44eefb26c (*: code format) if err != nil { klog.Errorf("Failed to calc volume snapshot backup, err: %v", err) return diff --git a/cmd/backup-manager/app/util/backup_size.go b/cmd/backup-manager/app/util/backup_size.go index 5f14cdae8f..32847514b2 100644 --- a/cmd/backup-manager/app/util/backup_size.go +++ b/cmd/backup-manager/app/util/backup_size.go @@ -53,6 +53,10 @@ const ( ListSnapMaxReturnResult = 10000 // This value can be between 1 and 50 due to aws service quota EbsApiConcurrency = 40 + + CalculateFullSize = "full" + CalculateIncremental = "incremental" + CalculateAll = "all" ) // CalcVolSnapBackupSize get snapshots from backup meta and then calc the backup size of snapshots. @@ -155,7 +159,7 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS return err ======= var snapSize uint64 - if level == backup.CalculateFullSize || level == backup.CalculateFullSize { + if level == CalculateFullSize || level == CalculateFullSize { snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId) if err != nil { return err @@ -164,6 +168,7 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS atomic.AddUint64(&apiReqCount, apiReq) >>>>>>> 33580d9dc (*: manual cp) } +<<<<<<< HEAD atomic.AddInt64(&fullBackupSize, int64(snapSize)) atomic.AddUint64(&apiReqCount, apiReq) @@ -173,6 +178,10 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS return err ======= if level == backup.CalculateAll || level == backup.CalculateIncremental { +======= + + if level == CalculateAll || level == CalculateIncremental { +>>>>>>> 44eefb26c (*: code format) volSnapshots, err := getVolSnapshots(volumeId) if err != nil { return err From b098c54a5e4e6dbf6164e77a509b01e111acead4 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 28 Jul 2023 00:14:43 +0800 Subject: [PATCH 4/7] *: fix typo Signed-off-by: BornChanger --- cmd/backup-manager/app/backup/manager.go | 4 ++++ cmd/backup-manager/app/util/backup_size.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index 8dedbd0a35..2edd6c9a37 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -318,11 +318,15 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d defer func() { // Calculate the backup size for ebs backup job even if it fails +<<<<<<< HEAD <<<<<<< HEAD if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize { fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider) ======= if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize && !bm.Initialize { +======= + if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize { +>>>>>>> f91a7b4e7 (*: fix typo) fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider, backup.Spec.CalcSizeLevel) >>>>>>> 44eefb26c (*: code format) if err != nil { diff --git a/cmd/backup-manager/app/util/backup_size.go b/cmd/backup-manager/app/util/backup_size.go index 32847514b2..53e47ee6fa 100644 --- a/cmd/backup-manager/app/util/backup_size.go +++ b/cmd/backup-manager/app/util/backup_size.go @@ -159,7 +159,7 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS return err ======= var snapSize uint64 - if level == CalculateFullSize || level == CalculateFullSize { + if level == CalculateAll || level == CalculateFullSize { snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId) if err != nil { return err From 6e873eb43968eaf24672612ab0153d01d8da4048 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 28 Jul 2023 20:40:12 +0800 Subject: [PATCH 5/7] *: address conflict Signed-off-by: BornChanger --- cmd/backup-manager/app/backup/manager.go | 9 --- cmd/backup-manager/app/util/backup_size.go | 65 +++---------------- docs/api-references/docs.md | 24 +++++++ docs/api-references/federation-docs.md | 12 ++++ manifests/crd.yaml | 9 +++ .../federation.pingcap.com_volumebackups.yaml | 3 + ...ion.pingcap.com_volumebackupschedules.yaml | 3 + manifests/crd/v1/pingcap.com_backups.yaml | 3 + .../crd/v1/pingcap.com_backupschedules.yaml | 6 ++ manifests/federation-crd.yaml | 6 ++ .../pingcap/v1alpha1/openapi_generated.go | 7 ++ pkg/apis/federation/pingcap/v1alpha1/types.go | 4 ++ .../pingcap/v1alpha1/openapi_generated.go | 7 ++ pkg/apis/pingcap/v1alpha1/types.go | 4 ++ pkg/backup/snapshotter/snapshotter_test.go | 5 +- 15 files changed, 101 insertions(+), 66 deletions(-) diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index 2edd6c9a37..e08767028f 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -318,17 +318,8 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d defer func() { // Calculate the backup size for ebs backup job even if it fails -<<<<<<< HEAD -<<<<<<< HEAD if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize { - fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider) -======= - if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize && !bm.Initialize { -======= - if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize { ->>>>>>> f91a7b4e7 (*: fix typo) fullBackupSize, incrementalBackupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider, backup.Spec.CalcSizeLevel) ->>>>>>> 44eefb26c (*: code format) if err != nil { klog.Errorf("Failed to calc volume snapshot backup, err: %v", err) return diff --git a/cmd/backup-manager/app/util/backup_size.go b/cmd/backup-manager/app/util/backup_size.go index 53e47ee6fa..49c6e55541 100644 --- a/cmd/backup-manager/app/util/backup_size.go +++ b/cmd/backup-manager/app/util/backup_size.go @@ -27,7 +27,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ebs" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/dustin/go-humanize" "github.com/pingcap/errors" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" @@ -47,11 +46,11 @@ import ( // interface CalcVolSnapBackupSize called by backup and backup clean. const ( - // This value can be between 5 and 1,000; if MaxResults is given a value larger than 1,000, only 1,000 results are returned. + // DescribeSnapMaxReturnResult can be between 5 and 1,000; if MaxResults is given a value larger than 1,000, only 1,000 results are returned. DescribeSnapMaxReturnResult = 1000 - // This value can be between 100 and 1,0000, and charge ~0.6$/1 million request + // ListSnapMaxReturnResult can be between 100 and 10,000, and charge ~0.6$/1 million request ListSnapMaxReturnResult = 10000 - // This value can be between 1 and 50 due to aws service quota + // EbsApiConcurrency can be between 1 and 50 due to aws service quota EbsApiConcurrency = 40 CalculateFullSize = "full" @@ -60,7 +59,7 @@ const ( ) // CalcVolSnapBackupSize get snapshots from backup meta and then calc the backup size of snapshots. -func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvider) (fullBackupSize int64, incrementalBackupSize int64, err error) { +func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvider, level string) (fullBackupSize int64, incrementalBackupSize int64, err error) { start := time.Now() // retrieves all snapshots from backup meta file volSnapshots, err := getSnapshotsFromBackupmeta(ctx, provider) @@ -72,7 +71,7 @@ func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvide return 0, 0, err } - fullBackupSize, incrementalBackupSize, err = calcBackupSize(ctx, volSnapshots) + fullBackupSize, incrementalBackupSize, err = calcBackupSize(ctx, volSnapshots, level) if err != nil { return 0, 0, err @@ -142,7 +141,7 @@ func getSnapshotsFromBackupmeta(ctx context.Context, provider v1alpha1.StoragePr } // calcBackupSize get a volume-snapshots backup size -func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupSize int64, incrementalBackupSize int64, err error) { +func calcBackupSize(ctx context.Context, volumes map[string]string, level string) (fullBackupSize int64, incrementalBackupSize int64, err error) { var apiReqCount, incrementalApiReqCount uint64 workerPool := util.NewWorkerPool(EbsApiConcurrency, "list snapshot size") @@ -153,11 +152,6 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS volumeId := vid // sort snapshots by timestamp workerPool.ApplyOnErrorGroup(eg, func() error { -<<<<<<< HEAD - snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId) - if err != nil { - return err -======= var snapSize uint64 if level == CalculateAll || level == CalculateFullSize { snapSize, apiReq, err := calculateSnapshotSize(volumeId, snapshotId) @@ -166,22 +160,9 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS } atomic.AddInt64(&fullBackupSize, int64(snapSize)) atomic.AddUint64(&apiReqCount, apiReq) ->>>>>>> 33580d9dc (*: manual cp) } -<<<<<<< HEAD - atomic.AddInt64(&fullBackupSize, int64(snapSize)) - atomic.AddUint64(&apiReqCount, apiReq) - -<<<<<<< HEAD - volSnapshots, err := getVolSnapshots(volumeId) - if err != nil { - return err -======= - if level == backup.CalculateAll || level == backup.CalculateIncremental { -======= if level == CalculateAll || level == CalculateIncremental { ->>>>>>> 44eefb26c (*: code format) volSnapshots, err := getVolSnapshots(volumeId) if err != nil { return err @@ -199,21 +180,7 @@ func calcBackupSize(ctx context.Context, volumes map[string]string) (fullBackupS } atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize)) atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq) ->>>>>>> 33580d9dc (*: manual cp) - } - prevSnapshotId, existed := getPrevSnapshotId(snapshotId, volSnapshots) - if !existed { - // if there is no previous snapshot, means it's the first snapshot, uses its full size as incremental size - atomic.AddInt64(&incrementalBackupSize, int64(snapSize)) - return nil } - klog.Infof("get previous snapshot %s of snapshot %s, volume %s", prevSnapshotId, snapshotId, volumeId) - incrementalSnapSize, incrementalApiReq, err := calculateChangedBlocksSize(volumeId, prevSnapshotId, snapshotId) - if err != nil { - return err - } - atomic.AddInt64(&incrementalBackupSize, int64(incrementalSnapSize)) - atomic.AddUint64(&incrementalApiReqCount, incrementalApiReq) return nil }) } @@ -260,31 +227,22 @@ func calculateSnapshotSize(volumeId, snapshotId string) (uint64, uint64, error) } nextToken = resp.NextToken } -<<<<<<< HEAD + klog.Infof("full snapshot size %s, num of ListSnapshotBlocks request %d, snapshot id %s, volume id %s", humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId) -======= - klog.Infof("full backup snapshot size %d bytes, num of ListSnapshotBlocks request %d, snapshot id %s, volume id %s", humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId) ->>>>>>> 33580d9dc (*: manual cp) + return snapshotSize, numApiReq, nil } // calculateChangedBlocksSize calculates changed blocks total size in bytes between two snapshots with common ancestry. -<<<<<<< HEAD func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uint64, uint64, error) { -======= -func calculateChangedBlocksSize(volumeId, preSnapshotId string, snapshotId string) (uint64, uint64, error) { ->>>>>>> 33580d9dc (*: manual cp) var numBlocks int var snapshotSize uint64 var numApiReq uint64 -<<<<<<< HEAD klog.Infof("start to calculate incremental snapshot size for %s, base on prev snapshot %s, volume id %s", snapshotId, preSnapshotId, volumeId) -======= - klog.Infof("the calc snapshot size for %s, base on prev snapshot %s, volume id %s", snapshotId, preSnapshotId, volumeId) ->>>>>>> 33580d9dc (*: manual cp) + ebsSession, err := util.NewEBSSession(util.CloudAPIConcurrency) if err != nil { klog.Errorf("new a ebs session failure.") @@ -319,12 +277,9 @@ func calculateChangedBlocksSize(volumeId, preSnapshotId string, snapshotId strin } nextToken = resp.NextToken } -<<<<<<< HEAD + klog.Infof("incremental snapshot size %s, num of api ListChangedBlocks request %d, snapshot id %s, volume id %s", humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId) -======= - klog.Infof("the total size of snapshot %d, num of api ListChangedBlocks request %d, snapshot id %s, volume id %s", humanize.Bytes(snapshotSize), numApiReq, snapshotId, volumeId) ->>>>>>> 33580d9dc (*: manual cp) return snapshotSize, numApiReq, nil } diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index e99cd74853..e2c6f6f2c4 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -270,6 +270,18 @@ bool +calcSizeLevel
+ +string + + + +(Optional) +

CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup

+ + + + federalVolumeBackupPhase
@@ -4036,6 +4048,18 @@ bool +calcSizeLevel
+ +string + + + +(Optional) +

CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup

+ + + + federalVolumeBackupPhase
 diff --git a/docs/api-references/federation-docs.md b/docs/api-references/federation-docs.md index bed43c216a..30976b8eb8 100644 --- a/docs/api-references/federation-docs.md +++ b/docs/api-references/federation-docs.md @@ -695,6 +695,18 @@ string

PriorityClassName of Backup Job Pods

+ + +calcSizeLevel
+ +string + + + +(Optional) +

CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup

+ +

VolumeBackupMemberStatus

diff --git a/manifests/crd.yaml b/manifests/crd.yaml index f0c020d00d..1ebb9a7608 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -503,6 +503,9 @@ spec: required: - cluster type: object + calcSizeLevel: + default: all + type: string cleanOption: properties: backoffEnabled: @@ -2126,6 +2129,9 @@ spec: required: - cluster type: object + calcSizeLevel: + default: all + type: string cleanOption: properties: backoffEnabled: @@ -3553,6 +3559,9 @@ spec: required: - cluster type: object + calcSizeLevel: + default: all + type: string cleanOption: properties: backoffEnabled: diff --git a/manifests/crd/federation/v1/federation.pingcap.com_volumebackups.yaml b/manifests/crd/federation/v1/federation.pingcap.com_volumebackups.yaml index 012e6874d3..0d3651da70 100644 --- a/manifests/crd/federation/v1/federation.pingcap.com_volumebackups.yaml +++ b/manifests/crd/federation/v1/federation.pingcap.com_volumebackups.yaml @@ -90,6 +90,9 @@ spec: sendCredToTikv: type: boolean type: object + calcSizeLevel: + default: all + type: string cleanPolicy: type: string env: diff --git a/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml b/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml index fbc7562e88..5a9825fc3c 100644 --- a/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml +++ b/manifests/crd/federation/v1/federation.pingcap.com_volumebackupschedules.yaml @@ -98,6 +98,9 @@ spec: sendCredToTikv: type: boolean type: object + calcSizeLevel: + default: all + type: string cleanPolicy: type: string env: diff --git a/manifests/crd/v1/pingcap.com_backups.yaml b/manifests/crd/v1/pingcap.com_backups.yaml index 3debf890af..81464a20c5 100644 --- a/manifests/crd/v1/pingcap.com_backups.yaml +++ b/manifests/crd/v1/pingcap.com_backups.yaml @@ -503,6 +503,9 @@ spec: required: - cluster type: object + calcSizeLevel: + default: all + type: string cleanOption: properties: backoffEnabled: diff --git a/manifests/crd/v1/pingcap.com_backupschedules.yaml b/manifests/crd/v1/pingcap.com_backupschedules.yaml index 30ff1d75f7..398482ecc9 100644 --- a/manifests/crd/v1/pingcap.com_backupschedules.yaml +++ b/manifests/crd/v1/pingcap.com_backupschedules.yaml @@ -478,6 +478,9 @@ spec: required: - cluster type: object + calcSizeLevel: + default: all + type: string cleanOption: properties: backoffEnabled: @@ -1905,6 +1908,9 @@ spec: required: - cluster type: object + calcSizeLevel: + default: all + type: string cleanOption: properties: backoffEnabled: diff --git a/manifests/federation-crd.yaml b/manifests/federation-crd.yaml index 3e44519cab..62f29ad377 100644 --- a/manifests/federation-crd.yaml +++ b/manifests/federation-crd.yaml @@ -90,6 +90,9 @@ spec: sendCredToTikv: type: boolean type: object + calcSizeLevel: + default: all + type: string cleanPolicy: type: string env: @@ -1143,6 +1146,9 @@ spec: sendCredToTikv: type: boolean type: object + calcSizeLevel: + default: all + type: string cleanPolicy: type: string env: diff --git a/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go index 2f7644d542..65381c34af 100644 --- a/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/federation/pingcap/v1alpha1/openapi_generated.go @@ -320,6 +320,13 @@ func schema_apis_federation_pingcap_v1alpha1_VolumeBackupMemberSpec(ref common.R Format: "", }, }, + "calcSizeLevel": { + SchemaProps: spec.SchemaProps{ + Description: "CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, diff --git a/pkg/apis/federation/pingcap/v1alpha1/types.go b/pkg/apis/federation/pingcap/v1alpha1/types.go index 35995880f6..e90c9195e4 100644 --- a/pkg/apis/federation/pingcap/v1alpha1/types.go +++ b/pkg/apis/federation/pingcap/v1alpha1/types.go @@ -113,6 +113,10 @@ type VolumeBackupMemberSpec struct { CleanPolicy pingcapv1alpha1.CleanPolicyType `json:"cleanPolicy,omitempty"` // PriorityClassName of Backup Job Pods PriorityClassName string `json:"priorityClassName,omitempty"` + // CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup + // +optional + // +kubebuilder:default="all" + CalcSizeLevel string `json:"calcSizeLevel,omitempty"` } // BRConfig contains config for BR diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index bd3b5d0f9c..126fb290f9 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -1067,6 +1067,13 @@ func schema_pkg_apis_pingcap_v1alpha1_BackupSpec(ref common.ReferenceCallback) c Format: "", }, }, + "calcSizeLevel": { + SchemaProps: spec.SchemaProps{ + Description: "CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup", + Type: []string{"string"}, + Format: "", + }, + }, "federalVolumeBackupPhase": { SchemaProps: spec.SchemaProps{ Description: "FederalVolumeBackupPhase indicates which phase to execute in federal volume backup", diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index f640768765..a35d56bca8 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -1949,6 +1949,10 @@ type BackupSpec struct { // LogStop indicates that will stop the log backup. // +optional LogStop bool `json:"logStop,omitempty"` + // CalcSizeLevel determines how to size calculation of snapshots for EBS volume snapshot backup + // +optional + // +kubebuilder:default="all" + CalcSizeLevel string `json:"calcSizeLevel,omitempty"` // FederalVolumeBackupPhase indicates which phase to execute in federal volume backup // +optional FederalVolumeBackupPhase FederalVolumeBackupPhase `json:"federalVolumeBackupPhase,omitempty"` diff --git a/pkg/backup/snapshotter/snapshotter_test.go b/pkg/backup/snapshotter/snapshotter_test.go index f44ab8b3b0..96152b7a25 100644 --- a/pkg/backup/snapshotter/snapshotter_test.go +++ b/pkg/backup/snapshotter/snapshotter_test.go @@ -429,8 +429,9 @@ func TestGenerateBackupMetadata(t *testing.T) { Annotations: make(map[string]string), }, Spec: v1alpha1.BackupSpec{ - Type: v1alpha1.BackupTypeFull, - Mode: v1alpha1.BackupModeVolumeSnapshot, + Type: v1alpha1.BackupTypeFull, + Mode: v1alpha1.BackupModeVolumeSnapshot, + CalcSizeLevel: "all", }, }, wantSSNil: false, From 292adf06a8d9f385bc99691cce6248aeffe52f64 Mon Sep 17 00:00:00 2001 From: BornChanger Date: Fri, 28 Jul 2023 20:58:24 +0800 Subject: [PATCH 6/7] *: code format Signed-off-by: BornChanger --- cmd/backup-manager/app/util/backup_size.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/backup-manager/app/util/backup_size.go b/cmd/backup-manager/app/util/backup_size.go index 49c6e55541..6582c42171 100644 --- a/cmd/backup-manager/app/util/backup_size.go +++ b/cmd/backup-manager/app/util/backup_size.go @@ -22,11 +22,10 @@ import ( "sync/atomic" "time" - "github.com/dustin/go-humanize" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ebs" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/dustin/go-humanize" "github.com/pingcap/errors" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" From 637583778d518bbc136036253a5061ea389ae4dd Mon Sep 17 00:00:00 2001 From: BornChanger Date: Mon, 31 Jul 2023 15:54:20 +0800 Subject: [PATCH 7/7] *: close issue https://github.com/pingcap/tidb-operator/issues/5212 Signed-off-by: BornChanger --- cmd/backup-manager/app/backup/manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go index e08767028f..f5d30dcdb8 100644 --- a/cmd/backup-manager/app/backup/manager.go +++ b/cmd/backup-manager/app/backup/manager.go @@ -332,6 +332,7 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d BackupSizeReadable: &backupSizeReadable, IncrementalBackupSize: &incrementalBackupSize, IncrementalBackupSizeReadable: &incrementalBackupSizeReadable, + TimeCompleted: &metav1.Time{Time: time.Now()}, } if err := bm.StatusUpdater.Update(backup, nil, updateStatus); err != nil {