From 55001680478de78dee45d0e4718d451f369fb950 Mon Sep 17 00:00:00 2001 From: Anthony Yeh Date: Thu, 13 Jun 2019 22:01:21 -0700 Subject: [PATCH] Add flag to wait for backup instead of starting up empty. Signed-off-by: Anthony Yeh --- doc/BackupAndRestore.md | 12 +++++-- go/vt/mysqlctl/backup.go | 4 +++ go/vt/mysqlctl/builtinbackupengine.go | 3 +- go/vt/mysqlctl/xtrabackupengine.go | 3 +- go/vt/vttablet/tabletmanager/action_agent.go | 4 +-- go/vt/vttablet/tabletmanager/restore.go | 35 ++++++++++++++++---- go/vt/vttablet/tabletmanager/rpc_backup.go | 2 +- go/vt/wrangler/testlib/backup_test.go | 2 +- test/backup.py | 30 +++++++++++++++-- 9 files changed, 76 insertions(+), 19 deletions(-) diff --git a/doc/BackupAndRestore.md b/doc/BackupAndRestore.md index efc5abad937..00f700e3111 100644 --- a/doc/BackupAndRestore.md +++ b/doc/BackupAndRestore.md @@ -168,8 +168,16 @@ to restore a backup to that tablet. As noted in the [Prerequisites](#prerequisites) section, the flag is generally enabled all of the time for all of the tablets in a shard. -If Vitess cannot find a backup in the Backup Storage system, it just -starts the vttablet as a new tablet. +By default, if Vitess cannot find a backup in the Backup Storage system, +the tablet will start up empty. This behavior allows you to bootstrap a new +shard before any backups exist. + +If the `-wait_for_backup_interval` flag is set to a value greater than zero, +the tablet will instead keep checking for a backup to appear at that interval. +This can be used to ensure tablets launched concurrently while an initial backup +is being seeded for the shard (e.g. uploaded from cold storage or created by +another tablet) will wait until the proper time and then pull the new backup +when it's ready. ``` sh vttablet ... -backup_storage_implementation=file \ diff --git a/go/vt/mysqlctl/backup.go b/go/vt/mysqlctl/backup.go index 1b8f498e6f5..b557638e077 100644 --- a/go/vt/mysqlctl/backup.go +++ b/go/vt/mysqlctl/backup.go @@ -55,6 +55,10 @@ var ( // ErrNoBackup is returned when there is no backup. ErrNoBackup = errors.New("no available backup") + // ErrNoCompleteBackup is returned when there is at least one backup, + // but none of them are complete. + ErrNoCompleteBackup = errors.New("backup(s) found but none are complete") + // ErrExistingDB is returned when there's already an active DB. ErrExistingDB = errors.New("skipping restore due to existing database") diff --git a/go/vt/mysqlctl/builtinbackupengine.go b/go/vt/mysqlctl/builtinbackupengine.go index 1ed80a8f12b..daa567dbde8 100644 --- a/go/vt/mysqlctl/builtinbackupengine.go +++ b/go/vt/mysqlctl/builtinbackupengine.go @@ -20,7 +20,6 @@ import ( "bufio" "context" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -543,7 +542,7 @@ func (be *BuiltinBackupEngine) ExecuteRestore( // There is at least one attempted backup, but none could be read. // This implies there is data we ought to have, so it's not safe to start // up empty. - return mysql.Position{}, errors.New("backup(s) found but none could be read, unsafe to start up empty, restart to retry restore") + return mysql.Position{}, ErrNoCompleteBackup } // Starting from here we won't be able to recover if we get stopped by a cancelled diff --git a/go/vt/mysqlctl/xtrabackupengine.go b/go/vt/mysqlctl/xtrabackupengine.go index 2399c1ffd40..2ed1ff27d4a 100644 --- a/go/vt/mysqlctl/xtrabackupengine.go +++ b/go/vt/mysqlctl/xtrabackupengine.go @@ -20,7 +20,6 @@ import ( "bufio" "context" "encoding/json" - "errors" "flag" "io" "io/ioutil" @@ -268,7 +267,7 @@ func (be *XtrabackupEngine) ExecuteRestore( // There is at least one attempted backup, but none could be read. // This implies there is data we ought to have, so it's not safe to start // up empty. - return zeroPosition, errors.New("backup(s) found but none could be read, unsafe to start up empty, restart to retry restore") + return zeroPosition, ErrNoCompleteBackup } // Starting from here we won't be able to recover if we get stopped by a cancelled diff --git a/go/vt/vttablet/tabletmanager/action_agent.go b/go/vt/vttablet/tabletmanager/action_agent.go index 98072227b75..3e34b89d9fc 100644 --- a/go/vt/vttablet/tabletmanager/action_agent.go +++ b/go/vt/vttablet/tabletmanager/action_agent.go @@ -303,9 +303,9 @@ func NewActionAgent( // - restoreFromBackup is not set: we initHealthCheck right away if *restoreFromBackup { go func() { - // restoreFromBackup wil just be a regular action + // restoreFromBackup will just be a regular action // (same as if it was triggered remotely) - if err := agent.RestoreData(batchCtx, logutil.NewConsoleLogger(), false /* deleteBeforeRestore */); err != nil { + if err := agent.RestoreData(batchCtx, logutil.NewConsoleLogger(), *waitForBackupInterval, false /* deleteBeforeRestore */); err != nil { println(fmt.Sprintf("RestoreFromBackup failed: %v", err)) log.Exitf("RestoreFromBackup failed: %v", err) } diff --git a/go/vt/vttablet/tabletmanager/restore.go b/go/vt/vttablet/tabletmanager/restore.go index 55a3a57285d..d01a3ad4fd6 100644 --- a/go/vt/vttablet/tabletmanager/restore.go +++ b/go/vt/vttablet/tabletmanager/restore.go @@ -19,6 +19,7 @@ package tabletmanager import ( "flag" "fmt" + "time" "vitess.io/vitess/go/vt/vterrors" @@ -37,15 +38,16 @@ import ( // It is only enabled if restore_from_backup is set. var ( - restoreFromBackup = flag.Bool("restore_from_backup", false, "(init restore parameter) will check BackupStorage for a recent backup at startup and start there") - restoreConcurrency = flag.Int("restore_concurrency", 4, "(init restore parameter) how many concurrent files to restore at once") + restoreFromBackup = flag.Bool("restore_from_backup", false, "(init restore parameter) will check BackupStorage for a recent backup at startup and start there") + restoreConcurrency = flag.Int("restore_concurrency", 4, "(init restore parameter) how many concurrent files to restore at once") + waitForBackupInterval = flag.Duration("wait_for_backup_interval", 0, "(init restore parameter) if this is greater than 0, instead of starting up empty when no backups are found, keep checking at this interval for a backup to appear") ) // RestoreData is the main entry point for backup restore. // It will either work, fail gracefully, or return // an error in case of a non-recoverable error. // It takes the action lock so no RPC interferes. -func (agent *ActionAgent) RestoreData(ctx context.Context, logger logutil.Logger, deleteBeforeRestore bool) error { +func (agent *ActionAgent) RestoreData(ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, deleteBeforeRestore bool) error { if err := agent.lock(ctx); err != nil { return err } @@ -53,10 +55,10 @@ func (agent *ActionAgent) RestoreData(ctx context.Context, logger logutil.Logger if agent.Cnf == nil { return fmt.Errorf("cannot perform restore without my.cnf, please restart vttablet with a my.cnf file specified") } - return agent.restoreDataLocked(ctx, logger, deleteBeforeRestore) + return agent.restoreDataLocked(ctx, logger, waitForBackupInterval, deleteBeforeRestore) } -func (agent *ActionAgent) restoreDataLocked(ctx context.Context, logger logutil.Logger, deleteBeforeRestore bool) error { +func (agent *ActionAgent) restoreDataLocked(ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, deleteBeforeRestore bool) error { // change type to RESTORE (using UpdateTabletFields so it's // always authorized) var originalType topodatapb.TabletType @@ -80,7 +82,28 @@ func (agent *ActionAgent) restoreDataLocked(ctx context.Context, logger logutil. localMetadata := agent.getLocalMetadataValues(originalType) tablet := agent.Tablet() dir := fmt.Sprintf("%v/%v", tablet.Keyspace, tablet.Shard) - pos, err := mysqlctl.Restore(ctx, agent.Cnf, agent.MysqlDaemon, dir, *restoreConcurrency, agent.hookExtraEnv(), localMetadata, logger, deleteBeforeRestore, topoproto.TabletDbName(tablet)) + + // Loop until a backup exists, unless we were told to give up immediately. + var pos mysql.Position + var err error + for { + pos, err = mysqlctl.Restore(ctx, agent.Cnf, agent.MysqlDaemon, dir, *restoreConcurrency, agent.hookExtraEnv(), localMetadata, logger, deleteBeforeRestore, topoproto.TabletDbName(tablet)) + if waitForBackupInterval == 0 { + break + } + // We only retry a specific set of errors. The rest we return immediately. + if err != mysqlctl.ErrNoBackup && err != mysqlctl.ErrNoCompleteBackup { + break + } + + log.Infof("No backup found. Waiting %v (from -wait_for_backup_interval flag) to check again.", waitForBackupInterval) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(waitForBackupInterval): + } + } + switch err { case nil: // Starting from here we won't be able to recover if we get stopped by a cancelled diff --git a/go/vt/vttablet/tabletmanager/rpc_backup.go b/go/vt/vttablet/tabletmanager/rpc_backup.go index f9a3d78fe3c..c4b9b06fca4 100644 --- a/go/vt/vttablet/tabletmanager/rpc_backup.go +++ b/go/vt/vttablet/tabletmanager/rpc_backup.go @@ -131,7 +131,7 @@ func (agent *ActionAgent) RestoreFromBackup(ctx context.Context, logger logutil. l := logutil.NewTeeLogger(logutil.NewConsoleLogger(), logger) // now we can run restore - err = agent.restoreDataLocked(ctx, l, true /* deleteBeforeRestore */) + err = agent.restoreDataLocked(ctx, l, 0 /* waitForBackupInterval */, true /* deleteBeforeRestore */) // re-run health check to be sure to capture any replication delay agent.runHealthCheckLocked() diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index ddd80965452..d4d68690ee2 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -178,7 +178,7 @@ func TestBackupRestore(t *testing.T) { RelayLogInfoPath: path.Join(root, "relay-log.info"), } - if err := destTablet.Agent.RestoreData(ctx, logutil.NewConsoleLogger(), false /* deleteBeforeRestore */); err != nil { + if err := destTablet.Agent.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */); err != nil { t.Fatalf("RestoreData failed: %v", err) } diff --git a/test/backup.py b/test/backup.py index 17973925c2a..57276cdf17c 100755 --- a/test/backup.py +++ b/test/backup.py @@ -227,6 +227,24 @@ def _restore(self, t, tablet_type='replica'): t.check_db_var('rpl_semi_sync_slave_enabled', 'OFF') t.check_db_status('rpl_semi_sync_slave_status', 'OFF') + def _restore_wait_for_backup(self, t, tablet_type='replica'): + """Erase mysql/tablet dir, then start tablet with wait_for_restore_interval.""" + self._reset_tablet_dir(t) + + xtra_args = [ + '-db-credentials-file', db_credentials_file, + '-wait_for_backup_interval', '1s', + ] + if use_xtrabackup: + xtra_args.extend(xtrabackup_args) + + t.start_vttablet(wait_for_state=None, + init_tablet_type=tablet_type, + init_keyspace='test_keyspace', + init_shard='0', + supports_backups=True, + extra_args=xtra_args) + def _reset_tablet_dir(self, t): """Stop mysql, delete everything including tablet dir, restart mysql.""" extra_args = ['-db-credentials-file', db_credentials_file] @@ -330,10 +348,11 @@ def _test_backup(self, tablet_type): test_backup will: - create a shard with master and replica1 only - run InitShardMaster + - bring up tablet_replica2 concurrently, telling it to wait for a backup - insert some data - take a backup - insert more data on the master - - bring up tablet_replica2 after the fact, let it restore the backup + - wait for tablet_replica2 to become SERVING - check all data is right (before+after backup data) - list the backup, remove it @@ -341,6 +360,10 @@ def _test_backup(self, tablet_type): tablet_type: 'replica' or 'rdonly'. """ + # bring up another replica concurrently, telling it to wait until a backup + # is available instead of starting up empty. + self._restore_wait_for_backup(tablet_replica2, tablet_type=tablet_type) + # insert data on master, wait for slave to get it tablet_master.mquery('vt_test_keyspace', self._create_vt_insert_test) self._insert_data(tablet_master, 1) @@ -358,8 +381,9 @@ def _test_backup(self, tablet_type): # insert more data on the master self._insert_data(tablet_master, 2) - # now bring up the other slave, letting it restore from backup. - self._restore(tablet_replica2, tablet_type=tablet_type) + # wait for tablet_replica2 to become serving (after restoring) + utils.pause('wait_for_backup') + tablet_replica2.wait_for_vttablet_state('SERVING') # check the new slave has the data self._check_data(tablet_replica2, 2, 'replica2 tablet getting data')