Merge pull request #5065 from planetscale/xtrabackup-stream-logs

xtrabackup: Better support for large datasets
vitessio · Aug 11, 2019 · 7e99841 · 7e99841
2 parents 95f1642 + 11391f1
commit 7e99841
Show file tree

Hide file tree

Showing 5 changed files with 534 additions and 173 deletions.
diff --git a/go/vt/mysqlctl/builtinbackupengine.go b/go/vt/mysqlctl/builtinbackupengine.go
@@ -339,7 +339,7 @@ func (be *BuiltinBackupEngine) ExecuteBackup(ctx context.Context, cnf *Mycnf, my
 }
 
 // backupFiles finds the list of files to backup, and creates the backup.
-func (be *BuiltinBackupEngine) backupFiles(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, replicationPosition mysql.Position, backupConcurrency int, hookExtraEnv map[string]string) (err error) {
+func (be *BuiltinBackupEngine) backupFiles(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, replicationPosition mysql.Position, backupConcurrency int, hookExtraEnv map[string]string) (finalErr error) {
 	// Get the files to backup.
 	fes, err := findFilesToBackup(cnf)
 	if err != nil {
@@ -381,8 +381,8 @@ func (be *BuiltinBackupEngine) backupFiles(ctx context.Context, cnf *Mycnf, mysq
 		return vterrors.Wrapf(err, "cannot add %v to backup", backupManifest)
 	}
 	defer func() {
-		if closeErr := wc.Close(); err == nil {
-			err = closeErr
+		if closeErr := wc.Close(); finalErr == nil {
+			finalErr = closeErr
 		}
 	}()
 
@@ -405,10 +405,9 @@ func (be *BuiltinBackupEngine) backupFiles(ctx context.Context, cnf *Mycnf, mysq
 }
 
 // backupFile backs up an individual file.
-func (be *BuiltinBackupEngine) backupFile(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, fe *FileEntry, name string, hookExtraEnv map[string]string) (err error) {
+func (be *BuiltinBackupEngine) backupFile(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, fe *FileEntry, name string, hookExtraEnv map[string]string) (finalErr error) {
 	// Open the source file for reading.
-	var source *os.File
-	source, err = fe.open(cnf, true)
+	source, err := fe.open(cnf, true)
 	if err != nil {
 		return err
 	}
@@ -427,11 +426,11 @@ func (be *BuiltinBackupEngine) backupFile(ctx context.Context, cnf *Mycnf, mysql
 	}
 	defer func(name, fileName string) {
 		if rerr := wc.Close(); rerr != nil {
-			if err != nil {
+			if finalErr != nil {
 				// We already have an error, just log this one.
 				logger.Errorf2(rerr, "failed to close file %v,%v", name, fe.Name)
 			} else {
-				err = rerr
+				finalErr = rerr
 			}
 		}
 	}(name, fe.Name)
@@ -577,10 +576,9 @@ func (be *BuiltinBackupEngine) restoreFiles(ctx context.Context, cnf *Mycnf, bh
 }
 
 // restoreFile restores an individual file.
-func (be *BuiltinBackupEngine) restoreFile(ctx context.Context, cnf *Mycnf, bh backupstorage.BackupHandle, fe *FileEntry, transformHook string, compress bool, name string, hookExtraEnv map[string]string) (err error) {
+func (be *BuiltinBackupEngine) restoreFile(ctx context.Context, cnf *Mycnf, bh backupstorage.BackupHandle, fe *FileEntry, transformHook string, compress bool, name string, hookExtraEnv map[string]string) (finalErr error) {
 	// Open the source file for reading.
-	var source io.ReadCloser
-	source, err = bh.ReadFile(ctx, name)
+	source, err := bh.ReadFile(ctx, name)
 	if err != nil {
 		return vterrors.Wrap(err, "can't open source file for reading")
 	}
@@ -593,11 +591,11 @@ func (be *BuiltinBackupEngine) restoreFile(ctx context.Context, cnf *Mycnf, bh b
 	}
 	defer func() {
 		if cerr := dstFile.Close(); cerr != nil {
-			if err != nil {
+			if finalErr != nil {
 				// We already have an error, just log this one.
 				log.Errorf("failed to close file %v: %v", name, cerr)
 			} else {
-				err = vterrors.Wrap(err, "failed to close destination file")
+				finalErr = vterrors.Wrap(cerr, "failed to close destination file")
 			}
 		}
 	}()
@@ -631,11 +629,11 @@ func (be *BuiltinBackupEngine) restoreFile(ctx context.Context, cnf *Mycnf, bh b
 		}
 		defer func() {
 			if cerr := gz.Close(); cerr != nil {
-				if err != nil {
+				if finalErr != nil {
 					// We already have an error, just log this one.
 					log.Errorf("failed to close gzip decompressor %v: %v", name, cerr)
 				} else {
-					err = vterrors.Wrap(err, "failed to close gzip decompressor")
+					finalErr = vterrors.Wrap(err, "failed to close gzip decompressor")
 				}
 			}
 		}()