ebs: calc full snapshot size and report backup size for failed backup

cmd/backup-manager/app/backup/manager.go

@@ -316,6 +316,27 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d
 	// run br binary to do the real job
 	backupErr := bm.backupData(ctx, backup, bm.StatusUpdater)
 
+	defer func() {
+		// Calculate the backup size for ebs backup job even if it fails
+		if bm.Mode == string(v1alpha1.BackupModeVolumeSnapshot) && !bm.Initialize {
+			backupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider)
+			if err != nil {
+				klog.Warningf("Failed to calc volume snapshot backup size %d bytes, %v", backupSize, err)
+				errs = append(errs, err)
+			}
+
+			backupSizeReadable := humanize.Bytes(uint64(backupSize))
+
+			updateStatus := &controller.BackupUpdateStatus{
+				BackupSize:         &backupSize,
+				BackupSizeReadable: &backupSizeReadable,
+			}
+
+			bm.StatusUpdater.Update(backup, nil,
+				updateStatus)
+		}
+	}()
+
 	if db != nil && oldTikvGCTimeDuration < tikvGCTimeDuration {
 		// use another context to revert `tikv_gc_life_time` back.
 		// `DefaultTerminationGracePeriodSeconds` for a pod is 30, so we use a smaller timeout value here.
@@ -328,6 +349,7 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d
 			}
 			errs = append(errs, err)
 			klog.Errorf("cluster %s reset tikv GC life time to %s failed, err: %s", bm, oldTikvGCTime, err)
+
 			uerr := bm.StatusUpdater.Update(backup, &v1alpha1.BackupCondition{
 				Type:    v1alpha1.BackupFailed,
 				Status:  corev1.ConditionTrue,
@@ -370,19 +392,9 @@ func (bm *Manager) performBackup(ctx context.Context, backup *v1alpha1.Backup, d
 			completeCondition = v1alpha1.VolumeBackupComplete
 			// In volume snapshot mode, commitTS have been updated according to the
 			// br command output, so we don't need to update it here.
-			backupSize, err := util.CalcVolSnapBackupSize(ctx, backup.Spec.StorageProvider)
-
-			if err != nil {
-				klog.Warningf("Failed to calc volume snapshot backup size %d bytes, %v", backupSize, err)
-			}
-
-			backupSizeReadable := humanize.Bytes(uint64(backupSize))
-
 			updateStatus = &controller.BackupUpdateStatus{
-				TimeStarted:        &metav1.Time{Time: started},
-				TimeCompleted:      &metav1.Time{Time: time.Now()},
-				BackupSize:         &backupSize,
-				BackupSizeReadable: &backupSizeReadable,
+				TimeStarted:   &metav1.Time{Time: started},
+				TimeCompleted: &metav1.Time{Time: time.Now()},
 			}
 		}
 	default:

cmd/backup-manager/app/clean/manager.go

@@ -89,11 +89,6 @@ func (bm *Manager) performCleanBackup(ctx context.Context, backup *v1alpha1.Back
 			klog.Errorf("delete backup %s for cluster %s backup failure", backup.Name, bm)
 		}
 
-		// update the next backup size
-		if nextNackup != nil {
-			bm.updateVolumeSnapshotBackupSize(ctx, nextNackup)
-		}
-
 	} else {
 		if backup.Spec.BR != nil {
 			err = bm.CleanBRRemoteBackupData(ctx, backup)

cmd/backup-manager/app/util/backup_size.go

@@ -66,14 +66,11 @@ func CalcVolSnapBackupSize(ctx context.Context, provider v1alpha1.StorageProvide
 		return 0, err
 	}
 
-	// get all snapshots per backup volume from aws
-	snapshots, err := getBackupVolSnapshots(volSnapshots)
-
 	if err != nil {
 		return 0, err
 	}
 
-	backupSize, err := calcBackupSize(ctx, volSnapshots, snapshots)
+	backupSize, err := calcBackupSize(ctx, volSnapshots)
 
 	if err != nil {
 		return 0, err
@@ -142,7 +139,7 @@ func getSnapshotsFromBackupmeta(ctx context.Context, provider v1alpha1.StoragePr
 	return volumeIDMap, nil
 }
 
-// getBackupVolSnapshots get a volue-snapshots map contains map[volumeId]{snapshot1, snapshot2, snapshot3}
+// getBackupVolSnapshots get a volume-snapshots map contains map[volumeId]{snapshot1, snapshot2, snapshot3}
 func getBackupVolSnapshots(volumes map[string]string) (map[string][]*ec2.Snapshot, error) {
 	volWithTheirSnapshots := make(map[string][]*ec2.Snapshot)
 
@@ -205,42 +202,22 @@ func getBackupVolSnapshots(volumes map[string]string) (map[string][]*ec2.Snapsho
 	return volWithTheirSnapshots, nil
 }
 
-// calcBackupSize get a volue-snapshots backup size
-func calcBackupSize(ctx context.Context, volumes map[string]string, snapshots map[string][]*ec2.Snapshot) (uint64, error) {
+// calcBackupSize get a volume-snapshots backup size
+func calcBackupSize(ctx context.Context, volumes map[string]string) (uint64, error) {
 	var backupSize uint64
 	var apiReqCount uint64
 
 	workerPool := util.NewWorkerPool(EbsApiConcurrency, "list snapshot size")
 	eg, _ := errgroup.WithContext(ctx)
 
-	for volumeId, id := range volumes {
-		volSnapshots := snapshots[volumeId]
+	for _, id := range volumes {
 		snapshotId := id
 		// sort snapshots by timestamp
 		workerPool.ApplyOnErrorGroup(eg, func() error {
-			// get prev snapshot backup
-			prevSnapshot, err := getPrevSnapshotId(snapshotId, volSnapshots)
-			if err != nil {
-				return err
-			}
-
-			// full/initial snapshot backup
-			if prevSnapshot == "" {
-				snapSize, apiReq, err := initialSnapshotSize(snapshotId)
-				if err != nil {
-					return err
-				}
-
-				atomic.AddUint64(&backupSize, snapSize)
-				atomic.AddUint64(&apiReqCount, apiReq)
-				return nil
-			}
-
-			snapSize, apiReq, err := changedBlocksSize(prevSnapshot, snapshotId)
+			snapSize, apiReq, err := calculateSnapshotSize(snapshotId)
 			if err != nil {
 				return err
 			}
-
 			atomic.AddUint64(&backupSize, snapSize)
 			atomic.AddUint64(&apiReqCount, apiReq)
 			return nil
@@ -257,9 +234,8 @@ func calcBackupSize(ctx context.Context, volumes map[string]string, snapshots ma
 	return backupSize, nil
 }
 
-// initialSnapshotSize calculate size of an initial snapshot in bytes by listing its blocks.
-// initial snapshot always a ful backup of volume
-func initialSnapshotSize(snapshotId string) (uint64, uint64, error) {
+// calculateSnapshotSize calculate size of an snapshot in bytes by listing its blocks.
+func calculateSnapshotSize(snapshotId string) (uint64, uint64, error) {
 	var snapshotSize uint64
 	var numApiReq uint64
 	ebsSession, err := util.NewEBSSession(util.CloudAPIConcurrency)

pkg/backup/util/remote.go

@@ -215,7 +215,7 @@ type BatchDeleteObjectsResult struct {
 	Errors  []ObjectError
 }
 
-// BatchDeleteObjects delete mutli objects
+// BatchDeleteObjects delete multi objects
 //
 // Depending on storage type, it use function 'BatchDeleteObjectsOfS3' or 'BatchDeleteObjectsConcurrently'
 func (b *StorageBackend) BatchDeleteObjects(ctx context.Context, objs []*blob.ListObject, opt v1alpha1.BatchDeleteOption) *BatchDeleteObjectsResult {