Debug file copy (zilliztech#233)

zhuwenxing · Nov 6, 2023 · 3ef7fcd · 3ef7fcd
1 parent b67cc9e
commit 3ef7fcd
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 1 deletion.
diff --git a/configs/backup.yaml b/configs/backup.yaml
@@ -49,6 +49,7 @@ backup:
     # which means if you set backup.parallelism = 2 backup.copydata.parallelism = 100, there will be 200 copy executing at the same time.
     # reduce it if blocks your storage's network bandwidth
     parallelism: 128
+  keepTempFiles: false
 
 restore:
   # Collection level parallelism to restore

diff --git a/core/backup_impl_restore_backup.go b/core/backup_impl_restore_backup.go
@@ -454,7 +454,7 @@ func (b *BackupContext) executeRestoreCollectionTask(ctx context.Context, backup
 	isSameBucket := b.milvusBucketName == backupBucketName
 	// clean the temporary file
 	defer func() {
-		if !isSameBucket {
+		if !isSameBucket && !b.params.BackupCfg.KeepTempFiles {
 			log.Info("Delete temporary file", zap.String("dir", tempDir))
 			err := b.getStorageClient().RemoveWithPrefix(ctx, b.milvusBucketName, tempDir)
 			if err != nil {
@@ -528,6 +528,7 @@ func (b *BackupContext) restorePartition(ctx context.Context, targetDBName, targ
 				if file == "" {
 					realFiles[i] = file
 				} else {
+					log.Debug("Copy temporary restore file", zap.String("from", file), zap.String("to", tempDir+file))
 					err := b.getStorageClient().Copy(ctx, backupBucketName, b.milvusBucketName, file, tempDir+file)
 					if err != nil {
 						log.Error("fail to copy backup date from backup bucket to restore target milvus bucket", zap.Error(err))

diff --git a/core/paramtable/params.go b/core/paramtable/params.go
@@ -37,6 +37,7 @@ type BackupConfig struct {
 	BackupParallelism         int
 	RestoreParallelism        int
 	BackupCopyDataParallelism int
+	KeepTempFiles             bool
 }
 
 func (p *BackupConfig) init(base *BaseTable) {
@@ -46,6 +47,7 @@ func (p *BackupConfig) init(base *BaseTable) {
 	p.initBackupParallelism()
 	p.initRestoreParallelism()
 	p.initBackupCopyDataParallelism()
+	p.initKeepTempFiles()
 }
 
 func (p *BackupConfig) initMaxSegmentGroupSize() {
@@ -71,6 +73,11 @@ func (p *BackupConfig) initBackupCopyDataParallelism() {
 	p.BackupCopyDataParallelism = size
 }
 
+func (p *BackupConfig) initKeepTempFiles() {
+	keepTempFiles := p.Base.LoadWithDefault("backup.keepTempFiles", "false")
+	p.KeepTempFiles, _ = strconv.ParseBool(keepTempFiles)
+}
+
 type MilvusConfig struct {
 	Base *BaseTable