xtrabackup implementation with testcases

bootstrap.sh

@@ -331,7 +331,7 @@ if [ "$BUILD_TESTS" == 1 ] ; then
  echo "Found MySQL 5.6+ installation in $VT_MYSQL_ROOT."
  ;;
 
- "MariaDB" | "MariaDB103" )
+ "MariaDB" | "MariaDB103")
  myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
  [[ "$myversion" =~ MariaDB ]] || fail "Couldn't find MariaDB in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
  echo "Found MariaDB installation in $VT_MYSQL_ROOT."

config/mycnf/default-fast.cnf

@@ -18,7 +18,7 @@ innodb_flush_log_at_trx_commit = 2
 innodb_flush_method = O_DIRECT
 innodb_lock_wait_timeout = 20
 innodb_log_buffer_size = 1M
-innodb_log_file_size = 1M
+innodb_log_file_size = 4M
 innodb_log_files_in_group = 2
 innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}}
 innodb_max_dirty_pages_pct = 75

docker/bootstrap/Dockerfile.mariadb

@@ -1,12 +1,18 @@
 FROM vitess/bootstrap:common
 
 # Install MariaDB 10
-RUN apt-get update \
+RUN apt-get update -y \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
  bzip2 \
  mariadb-server \
  libmariadbclient-dev \
- && rm -rf /var/lib/apt/lists/*
+ libdbd-mysql-perl \
+ rsync \
+ libev4 \
+ && rm -rf /var/lib/apt/lists/* \
+ && wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb \
+ && dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb \
+ && rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.mysql56

@@ -3,9 +3,12 @@ FROM vitess/bootstrap:common
 # Install MySQL 5.6
 RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver pool.sks-keyservers.net 5072E1F5 && break; done && \
  add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.6' && \
- apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev && \
- rm -rf /var/lib/apt/lists/*
+ apt-get update -y && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 && \
+ rm -rf /var/lib/apt/lists/* && \
+ wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
+ dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
+ rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.mysql57

@@ -4,9 +4,12 @@ FROM vitess/bootstrap:common
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && \
  for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 5072E1F5 && break; done && \
  add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.7' && \
- apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev && \
- rm -rf /var/lib/apt/lists/*
+ apt-get update -y && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 && \
+ rm -rf /var/lib/apt/lists/* && \
+ wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
+ dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
+ rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.mysql80

@@ -3,9 +3,12 @@ FROM vitess/bootstrap:common
 # Install MySQL 5.7
 RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 8C718D3B5072E1F5 && break; done && \
  add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-8.0' && \
- apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev && \
- rm -rf /var/lib/apt/lists/*
+ apt-get update -y && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 && \
+ rm -rf /var/lib/apt/lists/* && \
+ wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-8.0.4/binary/debian/stretch/x86_64/percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb && \
+ dpkg -i percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb && \
+ rm -f percona-xtrabackup-80_8.0.4-1.stretch_amd64.deb
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

go/vt/mysqlctl/backup.go

@@ -19,7 +19,6 @@ package mysqlctl
 import (
  "errors"
  "flag"
- "fmt"
  "os"
  "path/filepath"
  "strings"
@@ -31,6 +30,8 @@ import (
  "vitess.io/vitess/go/vt/log"
  "vitess.io/vitess/go/vt/logutil"
  "vitess.io/vitess/go/vt/mysqlctl/backupstorage"
+ "vitess.io/vitess/go/vt/proto/vtrpc"
+ "vitess.io/vitess/go/vt/vterrors"
 )
 
 // This file handles the backup and restore related code
@@ -86,17 +87,17 @@ func Backup(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.
  // Start the backup with the BackupStorage.
  bs, err := backupstorage.GetBackupStorage()
  if err != nil {
- return err
+ return vterrors.Wrap(err, "unable to get backup storage")
  }
  defer bs.Close()
  bh, err := bs.StartBackup(ctx, dir, name)
  if err != nil {
- return fmt.Errorf("StartBackup failed: %v", err)
+ return vterrors.Wrap(err, "StartBackup failed")
  }
 
  be, err := GetBackupEngine()
  if err != nil {
- return fmt.Errorf("Failed to find backup engine: %v", err)
+ return vterrors.Wrap(err, "failed to find backup engine")
  }
 
  // Take the backup, and either AbortBackup or EndBackup.
@@ -132,15 +133,15 @@ func Backup(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.
 func checkNoDB(ctx context.Context, mysqld MysqlDaemon, dbName string) (bool, error) {
  qr, err := mysqld.FetchSuperQuery(ctx, "SHOW DATABASES")
  if err != nil {
- return false, fmt.Errorf("checkNoDB failed: %v", err)
+ return false, vterrors.Wrap(err, "checkNoDB failed")
  }
 
  backtickDBName := sqlescape.EscapeID(dbName)
  for _, row := range qr.Rows {
  if row[0].ToString() == dbName {
  tableQr, err := mysqld.FetchSuperQuery(ctx, "SHOW TABLES FROM "+backtickDBName)
  if err != nil {
- return false, fmt.Errorf("checkNoDB failed: %v", err)
+ return false, vterrors.Wrap(err, "checkNoDB failed")
  }
  if len(tableQr.Rows) == 0 {
  // no tables == empty db, all is well
@@ -171,7 +172,7 @@ func removeExistingFiles(cnf *Mycnf) error {
  }
  for name, path := range paths {
  if path == "" {
- return fmt.Errorf("can't remove existing files: %v is unknown", name)
+ return vterrors.Errorf(vtrpc.Code_UNKNOWN, "can't remove existing files: %v is unknown", name)
  }
 
  if strings.HasSuffix(name, ".*") {
@@ -181,11 +182,11 @@ func removeExistingFiles(cnf *Mycnf) error {
  log.Infof("Restore: removing files in %v (%v)", name, path)
  matches, err := filepath.Glob(path)
  if err != nil {
- return fmt.Errorf("can't expand path glob %q: %v", path, err)
+ return vterrors.Wrapf(err, "can't expand path glob %q", path)
  }
  for _, match := range matches {
  if err := os.Remove(match); err != nil {
- return fmt.Errorf("can't remove existing file from %v (%v): %v", name, match, err)
+ return vterrors.Wrapf(err, "can't remove existing file from %v (%v)", name, match)
  }
  }
  continue
@@ -198,7 +199,7 @@ func removeExistingFiles(cnf *Mycnf) error {
  }
  log.Infof("Restore: removing files in %v (%v)", name, path)
  if err := os.RemoveAll(path); err != nil {
- return fmt.Errorf("can't remove existing files in %v (%v): %v", name, path, err)
+ return vterrors.Wrapf(err, "can't remove existing files in %v (%v)", name, path)
  }
  }
  return nil
@@ -251,15 +252,15 @@ func Restore(
 
  bhs, err := bs.ListBackups(ctx, dir)
  if err != nil {
- return mysql.Position{}, fmt.Errorf("ListBackups failed: %v", err)
+ return mysql.Position{}, vterrors.Wrap(err, "ListBackups failed")
  }
 
  if len(bhs) == 0 {
  // There are no backups (not even broken/incomplete ones).
- logger.Errorf("No backup to restore on BackupStorage for directory %v. Starting up empty.", dir)
+ logger.Errorf("no backup to restore on BackupStorage for directory %v. Starting up empty.", dir)
  // Since this is an empty database make sure we start replication at the beginning
  if err = mysqld.ResetReplication(ctx); err == nil {
- logger.Errorf("Error reseting slave replication: %v. Continuing", err)
+ logger.Errorf("error reseting slave replication: %v. Continuing", err)
  err = ErrNoBackup
  }
 
@@ -271,7 +272,7 @@ func Restore(
 
  be, err := GetBackupEngine()
  if err != nil {
- return mysql.Position{}, fmt.Errorf("Failed to find backup engine: %v", err)
+ return mysql.Position{}, vterrors.Wrap(err, "Failed to find backup engine")
  }
  if rval, err = be.ExecuteRestore(ctx, cnf, mysqld, logger, dir, bhs, restoreConcurrency, hookExtraEnv); err != nil {
  return rval, err
@@ -293,7 +294,7 @@ func Restore(
 
  logger.Infof("Restore: running mysql_upgrade")
  if err := mysqld.RunMysqlUpgrade(); err != nil {
- return mysql.Position{}, fmt.Errorf("mysql_upgrade failed: %v", err)
+ return mysql.Position{}, vterrors.Wrap(err, "mysql_upgrade failed")
  }
 
  // Populate local_metadata before starting without --skip-networking,

go/vt/mysqlctl/backupengine.go

@@ -19,17 +19,17 @@ package mysqlctl
 import (
  "context"
  "flag"
- "fmt"
 
  "vitess.io/vitess/go/mysql"
  "vitess.io/vitess/go/vt/logutil"
  "vitess.io/vitess/go/vt/mysqlctl/backupstorage"
+ "vitess.io/vitess/go/vt/proto/vtrpc"
+ "vitess.io/vitess/go/vt/vterrors"
 )
 
 var (
- // BackupEngineImplementation is the implementation to use
- // for BackupEngine. Exported for test purposes.
- BackupEngineImplementation = flag.String("backup_engine_implementation", "builtin", "which implementation to use for the backup storage engine")
+ // BackupEngineImplementation is the implementation to use for BackupEngine
+ backupEngineImplementation = flag.String("backup_engine_implementation", builtin, "which implementation to use for the backup method, builtin or xtrabackup")
 )
 
 // BackupEngine is the interface to the backup engine
@@ -44,9 +44,9 @@ var BackupEngineMap = make(map[string]BackupEngine)
 // GetBackupEngine returns the current BackupEngine implementation.
 // Should be called after flags have been initialized.
 func GetBackupEngine() (BackupEngine, error) {
- be, ok := BackupEngineMap[*BackupEngineImplementation]
+ be, ok := BackupEngineMap[*backupEngineImplementation]
  if !ok {
- return nil, fmt.Errorf("no registered implementation of BackupEngine")
+ return nil, vterrors.New(vtrpc.Code_NOT_FOUND, "no registered implementation of BackupEngine")
  }
  return be, nil
 }