Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

xtrabackup: Better support for large datasets #5065

Merged
merged 9 commits into from
Aug 11, 2019
Merged

xtrabackup: Better support for large datasets #5065

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
b56856b
xtrabackup: Stream stderr to logs.
enisoc Aug 8, 2019
36fef02
Scan lines to send to logger instead of direct write.
enisoc Aug 9, 2019
4cb274d
xtrabackup: Add verbose flag to tar.
enisoc Aug 9, 2019
93074c5
Use move-back instead of copy-back.
enisoc Aug 9, 2019
fdce969
Clean up Close() error handling.
enisoc Aug 10, 2019
af13447
Add verbose flag to xbstream extraction command.
enisoc Aug 10, 2019
b185573
Add data striping for xtrabackup.
enisoc Aug 10, 2019
870a07f
Add unit test for data striping.
enisoc Aug 10, 2019
11391f1
Test data striping in integration test.
enisoc Aug 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 13 additions & 15 deletions go/vt/mysqlctl/builtinbackupengine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ func (be *BuiltinBackupEngine) ExecuteBackup(ctx context.Context, cnf *Mycnf, my
}

// backupFiles finds the list of files to backup, and creates the backup.
func (be *BuiltinBackupEngine) backupFiles(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, replicationPosition mysql.Position, backupConcurrency int, hookExtraEnv map[string]string) (err error) {
func (be *BuiltinBackupEngine) backupFiles(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, replicationPosition mysql.Position, backupConcurrency int, hookExtraEnv map[string]string) (finalErr error) {
// Get the files to backup.
fes, err := findFilesToBackup(cnf)
if err != nil {
Expand Down Expand Up @@ -381,8 +381,8 @@ func (be *BuiltinBackupEngine) backupFiles(ctx context.Context, cnf *Mycnf, mysq
return vterrors.Wrapf(err, "cannot add %v to backup", backupManifest)
}
defer func() {
if closeErr := wc.Close(); err == nil {
err = closeErr
if closeErr := wc.Close(); finalErr == nil {
finalErr = closeErr
}
}()

Expand All @@ -405,10 +405,9 @@ func (be *BuiltinBackupEngine) backupFiles(ctx context.Context, cnf *Mycnf, mysq
}

// backupFile backs up an individual file.
func (be *BuiltinBackupEngine) backupFile(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, fe *FileEntry, name string, hookExtraEnv map[string]string) (err error) {
func (be *BuiltinBackupEngine) backupFile(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, fe *FileEntry, name string, hookExtraEnv map[string]string) (finalErr error) {
// Open the source file for reading.
var source *os.File
source, err = fe.open(cnf, true)
source, err := fe.open(cnf, true)
if err != nil {
return err
}
Expand All @@ -427,11 +426,11 @@ func (be *BuiltinBackupEngine) backupFile(ctx context.Context, cnf *Mycnf, mysql
}
defer func(name, fileName string) {
if rerr := wc.Close(); rerr != nil {
if err != nil {
if finalErr != nil {
// We already have an error, just log this one.
logger.Errorf2(rerr, "failed to close file %v,%v", name, fe.Name)
} else {
err = rerr
finalErr = rerr
}
}
}(name, fe.Name)
Expand Down Expand Up @@ -577,10 +576,9 @@ func (be *BuiltinBackupEngine) restoreFiles(ctx context.Context, cnf *Mycnf, bh
}

// restoreFile restores an individual file.
func (be *BuiltinBackupEngine) restoreFile(ctx context.Context, cnf *Mycnf, bh backupstorage.BackupHandle, fe *FileEntry, transformHook string, compress bool, name string, hookExtraEnv map[string]string) (err error) {
func (be *BuiltinBackupEngine) restoreFile(ctx context.Context, cnf *Mycnf, bh backupstorage.BackupHandle, fe *FileEntry, transformHook string, compress bool, name string, hookExtraEnv map[string]string) (finalErr error) {
// Open the source file for reading.
var source io.ReadCloser
source, err = bh.ReadFile(ctx, name)
source, err := bh.ReadFile(ctx, name)
if err != nil {
return vterrors.Wrap(err, "can't open source file for reading")
}
Expand All @@ -593,11 +591,11 @@ func (be *BuiltinBackupEngine) restoreFile(ctx context.Context, cnf *Mycnf, bh b
}
defer func() {
if cerr := dstFile.Close(); cerr != nil {
if err != nil {
if finalErr != nil {
// We already have an error, just log this one.
log.Errorf("failed to close file %v: %v", name, cerr)
} else {
err = vterrors.Wrap(err, "failed to close destination file")
finalErr = vterrors.Wrap(cerr, "failed to close destination file")
}
}
}()
Expand Down Expand Up @@ -631,11 +629,11 @@ func (be *BuiltinBackupEngine) restoreFile(ctx context.Context, cnf *Mycnf, bh b
}
defer func() {
if cerr := gz.Close(); cerr != nil {
if err != nil {
if finalErr != nil {
// We already have an error, just log this one.
log.Errorf("failed to close gzip decompressor %v: %v", name, cerr)
} else {
err = vterrors.Wrap(err, "failed to close gzip decompressor")
finalErr = vterrors.Wrap(err, "failed to close gzip decompressor")
}
}
}()
Expand Down
Loading