From af81ebc10703ed69096bc09c5bca37be4a4344b1 Mon Sep 17 00:00:00 2001
From: Anthony Yeh <enisoc@planetscale.com>
Date: Tue, 4 Jun 2019 16:14:50 -0700
Subject: [PATCH 1/8] vtbackup: Document and clean up.

Signed-off-by: Anthony Yeh <enisoc@planetscale.com>
---
 go/cmd/vtbackup/vtbackup.go | 135 ++++++++++++++++++++++++++----------
 1 file changed, 100 insertions(+), 35 deletions(-)

diff --git a/go/cmd/vtbackup/vtbackup.go b/go/cmd/vtbackup/vtbackup.go
index 0fdb663accc..d0ad5f1143e 100644
--- a/go/cmd/vtbackup/vtbackup.go
+++ b/go/cmd/vtbackup/vtbackup.go
@@ -14,16 +14,54 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// vt backup job: Restore a Backup and Takes a new backup
+/*
+vtbackup is a batch command to perform a single pass of backup maintenance for a shard.
+
+When run periodically for each shard, vtbackup can ensure these configurable policies:
+* There is always a recent backup for the shard.
+* Old backups for the shard are removed.
+
+Whatever system launches vtbackup is responsible for the following:
+* Running vtbackup with similar flags that would be used for a vttablet in the
+  target shard to be backed up.
+* Running mysqlctld alongside vtbackup, as it would be alongside vttablet.
+* Provisioning as much disk space for vtbackup as would be given to vttablet.
+  The data directory MUST be empty at startup. Do NOT reuse a persistent disk.
+* Running vtbackup periodically for each shard, for each backup storage location.
+* Ensuring that at most one instance runs at a time for a given pair of shard
+  and backup storage location.
+* Retrying vtbackup if it fails.
+* Alerting human operators if the failure is persistent.
+
+The process vtbackup follows to take a new backup is as follows:
+1. Restore from the most recent backup.
+2. Start a mysqld instance (but no vttablet) from the restored data.
+3. Instruct mysqld to connect to the current shard master and replicate any
+   transactions that are new since the last backup.
+4. Wait until replication is caught up to the master.
+5. Stop mysqld and take a new backup.
+
+Aside from additional replication load while vtbackup's mysqld catches up on
+new transactions, the shard should be otherwise unaffected. Existing tablets
+will continue to serve, and no new tablets will appear in topology, meaning no
+query traffic will ever be routed to vtbackup's mysqld. This silent operation
+mode helps make backups minimally disruptive to serving capacity and orthogonal
+to the handling of the query path.
+
+The command-line parameters to vtbackup specify a policy for when a new backup
+is needed, and when old backups should be removed. If the existing backups
+already satisfy the policy, then vtbackup will do nothing and return success
+immediately.
+*/
 package main
 
 import (
 	"context"
 	"flag"
 	"fmt"
-	"strconv"
 	"time"
 
+	"vitess.io/vitess/go/exit"
 	"vitess.io/vitess/go/mysql"
 	"vitess.io/vitess/go/vt/dbconfigs"
 	"vitess.io/vitess/go/vt/log"
@@ -41,18 +79,25 @@ var (
 	initShard                = flag.String("init_shard", "", "(init parameter) shard to use for this tablet")
 	tabletPath               = flag.String("tablet-path", "", "tablet alias")
 	concurrency              = flag.Int("concurrency", 4, "(init restore parameter) how many concurrent files to restore at once")
-	acceptableReplicationLag = flag.Duration("acceptable_replication_lag", 0, "set what the is the acceptable replication lag to wait for before a backup is taken")
+	acceptableReplicationLag = flag.Duration("acceptable_replication_lag", 1*time.Second, "Wait until replication lag is less than or equal to this value before taking a new backup")
+	timeout                  = flag.Duration("timeout", 1*time.Hour, "Overall timeout for this vtbackup run")
 )
 
 func main() {
+	defer exit.Recover()
+
 	dbconfigs.RegisterFlags(dbconfigs.All...)
 	mysqlctl.RegisterFlags()
 
 	servenv.ParseFlags("vtbackup")
 
+	ctx, cancel := context.WithTimeout(context.Background(), *timeout)
+	defer cancel()
+
 	tabletAlias, err := topoproto.ParseTabletAlias(*tabletPath)
 	if err != nil {
-		log.Exitf("failed to parse -tablet-path: %v", err)
+		log.Errorf("failed to parse -tablet-path: %v", err)
+		exit.Return(1)
 	}
 
 	dbName := *initDbNameOverride
@@ -62,71 +107,91 @@ func main() {
 
 	var mycnf *mysqlctl.Mycnf
 	var socketFile string
-	extraEnv := map[string]string{"TABLET_ALIAS": strconv.FormatUint(uint64(tabletAlias.Uid), 10)}
+	extraEnv := map[string]string{
+		"TABLET_ALIAS": topoproto.TabletAliasString(tabletAlias),
+	}
 
-	if !dbconfigs.HasConnectionParams() {
+	if dbconfigs.HasConnectionParams() {
+		log.Info("connection parameters were specified. Not loading my.cnf.")
+	} else {
 		var err error
 		if mycnf, err = mysqlctl.NewMycnfFromFlags(tabletAlias.Uid); err != nil {
-			log.Exitf("mycnf read failed: %v", err)
+			log.Errorf("mycnf read failed: %v", err)
+			exit.Return(1)
 		}
 		socketFile = mycnf.SocketFile
-	} else {
-		log.Info("connection parameters were specified. Not loading my.cnf.")
 	}
 
 	dbcfgs, err := dbconfigs.Init(socketFile)
 	if err != nil {
-		log.Warning(err)
+		log.Errorf("can't initialize dbconfigs: %v", err)
+		exit.Return(1)
 	}
 
 	topoServer := topo.Open()
 	mysqld := mysqlctl.NewMysqld(dbcfgs)
 	dir := fmt.Sprintf("%v/%v", *initKeyspace, *initShard)
 
-	pos, err := mysqlctl.Restore(context.Background(), mycnf, mysqld, dir, *concurrency, extraEnv, map[string]string{}, logutil.NewConsoleLogger(), true, dbName)
+	log.Infof("Restoring latest backup from directory %v", dir)
+	pos, err := mysqlctl.Restore(ctx, mycnf, mysqld, dir, *concurrency, extraEnv, map[string]string{}, logutil.NewConsoleLogger(), true, dbName)
 	switch err {
 	case nil:
-		// Horray Evenything worked
-		// We have restored a backup ( If one existed, now make sure replicatoin is started
-		if err := resetReplication(context.Background(), pos, mysqld); err != nil {
-			log.Fatalf("Error Starting Replication %v", err)
-		}
-
+		log.Info("Successfully restored from backup at replication position %v", pos)
 	case mysqlctl.ErrNoBackup:
-		// No-op, starting with empty database.
+		log.Error("No backup found. Not starting up empty since -initial_backup flag was not enabled.")
+		exit.Return(1)
 	case mysqlctl.ErrExistingDB:
-		// No-op, assuming we've just restarted.  Note the
-		// replication reporter may restart replication at the
-		// next health check if it thinks it should. We do not
-		// alter replication here.
+		log.Error("Can't run vtbackup because data directory is not empty.")
+		exit.Return(1)
 	default:
-		log.Fatalf("Error restoring backup: %v", err)
+		log.Errorf("Error restoring from backup: %v", err)
+		exit.Return(1)
 	}
 
-	// We have restored a backup ( If one existed, now make sure replicatoin is started
-	if err := startReplication(context.Background(), pos, mysqld, topoServer); err != nil {
-		log.Fatalf("Error Starting Replication %v", err)
+	// We have restored a backup. Now start replication.
+	if err := resetReplication(ctx, pos, mysqld); err != nil {
+		log.Errorf("Error resetting replication %v", err)
+		exit.Return(1)
+	}
+	if err := startReplication(ctx, pos, mysqld, topoServer); err != nil {
+		log.Errorf("Error starting replication %v", err)
+		exit.Return(1)
 	}
 
+	// Wait for replication to catch up.
+	waitStartTime := time.Now()
 	for {
+		time.Sleep(time.Second)
+
+		// Check if the context is still good.
+		if err := ctx.Err(); err != nil {
+			log.Errorf("Timed out waiting for replication to catch up to within %v.", *acceptableReplicationLag)
+			exit.Return(1)
+		}
+
 		status, statusErr := mysqld.SlaveStatus()
 		if statusErr != nil {
-			log.Warning("Error getting Slave Status (%v)", statusErr)
-		} else if time.Duration(status.SecondsBehindMaster)*time.Second <= *acceptableReplicationLag {
+			log.Warningf("Error getting replication status: %v", statusErr)
+			continue
+		}
+		if time.Duration(status.SecondsBehindMaster)*time.Second <= *acceptableReplicationLag {
+			// We're caught up on replication.
+			log.Infof("Replication caught up to within %v after %v", *acceptableReplicationLag, time.Since(waitStartTime))
 			break
 		}
 		if !status.SlaveRunning() {
-			log.Warning("Slave has stopped before backup could be taken")
-			startReplication(context.Background(), pos, mysqld, topoServer)
+			log.Warning("Replication has stopped before backup could be taken. Trying to restart replication.")
+			if err := startReplication(ctx, pos, mysqld, topoServer); err != nil {
+				log.Warningf("Failed to restart replication: %v", err)
+			}
 		}
-		time.Sleep(time.Second)
 	}
 
-	// now we can run the backup
+	// Now we can take a new backup.
 	name := fmt.Sprintf("%v.%v", time.Now().UTC().Format("2006-01-02.150405"), topoproto.TabletAliasString(tabletAlias))
-	returnErr := mysqlctl.Backup(context.Background(), mycnf, mysqld, logutil.NewConsoleLogger(), dir, name, *concurrency, extraEnv)
-	if returnErr != nil {
-		log.Fatalf("Error taking backup: %v", returnErr)
+	if err := mysqlctl.Backup(ctx, mycnf, mysqld, logutil.NewConsoleLogger(), dir, name, *concurrency, extraEnv); err != nil {
+		log.Errorf("Error taking backup: %v", err)
+		exit.Return(1)
 	}
 }
 

From 9a43d1074b34d8ccc8bb31ba1d835bc541c01f6b Mon Sep 17 00:00:00 2001
From: Anthony Yeh <enisoc@planetscale.com>
Date: Tue, 4 Jun 2019 17:12:34 -0700
Subject: [PATCH 2/8] vtbackup: Incorporate mysqlctld behavior so that's not
 needed.

Signed-off-by: Anthony Yeh <enisoc@planetscale.com>
---
 go/cmd/vtbackup/vtbackup.go | 100 ++++++++++++++++++++++++------------
 1 file changed, 67 insertions(+), 33 deletions(-)

diff --git a/go/cmd/vtbackup/vtbackup.go b/go/cmd/vtbackup/vtbackup.go
index d0ad5f1143e..e8ce0c499c3 100644
--- a/go/cmd/vtbackup/vtbackup.go
+++ b/go/cmd/vtbackup/vtbackup.go
@@ -22,9 +22,8 @@ When run periodically for each shard, vtbackup can ensure these configurable pol
 * Old backups for the shard are removed.
 
 Whatever system launches vtbackup is responsible for the following:
-* Running vtbackup with similar flags that would be used for a vttablet in the
-  target shard to be backed up.
-* Running mysqlctld alongside vtbackup, as it would be alongside vttablet.
+* Running vtbackup with similar flags that would be used for a vttablet and
+  mysqlctld in the target shard to be backed up.
 * Provisioning as much disk space for vtbackup as would be given to vttablet.
   The data directory MUST be empty at startup. Do NOT reuse a persistent disk.
 * Running vtbackup periodically for each shard, for each backup storage location.
@@ -57,8 +56,12 @@ package main
 
 import (
 	"context"
+	"crypto/rand"
 	"flag"
 	"fmt"
+	"math"
+	"math/big"
+	"os"
 	"time"
 
 	"vitess.io/vitess/go/exit"
@@ -67,6 +70,7 @@ import (
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/logutil"
 	"vitess.io/vitess/go/vt/mysqlctl"
+	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/servenv"
 	"vitess.io/vitess/go/vt/topo"
 	"vitess.io/vitess/go/vt/topo/topoproto"
@@ -74,17 +78,26 @@ import (
 )
 
 var (
-	initDbNameOverride       = flag.String("init_db_name_override", "", "(init parameter) override the name of the db used by vttablet")
-	initKeyspace             = flag.String("init_keyspace", "", "(init parameter) keyspace to use for this tablet")
-	initShard                = flag.String("init_shard", "", "(init parameter) shard to use for this tablet")
-	tabletPath               = flag.String("tablet-path", "", "tablet alias")
-	concurrency              = flag.Int("concurrency", 4, "(init restore parameter) how many concurrent files to restore at once")
+	// vtbackup-specific flags
 	acceptableReplicationLag = flag.Duration("acceptable_replication_lag", 1*time.Second, "Wait until replication lag is less than or equal to this value before taking a new backup")
 	timeout                  = flag.Duration("timeout", 1*time.Hour, "Overall timeout for this vtbackup run")
+
+	// vttablet-like flags
+	initDbNameOverride = flag.String("init_db_name_override", "", "(init parameter) override the name of the db used by vttablet")
+	initKeyspace       = flag.String("init_keyspace", "", "(init parameter) keyspace to use for this tablet")
+	initShard          = flag.String("init_shard", "", "(init parameter) shard to use for this tablet")
+	concurrency        = flag.Int("concurrency", 4, "(init restore parameter) how many concurrent files to restore at once")
+
+	// mysqlctld-like flags
+	mysqlPort     = flag.Int("mysql_port", 3306, "mysql port")
+	mysqlSocket   = flag.String("mysql_socket", "", "path to the mysql socket")
+	mysqlTimeout  = flag.Duration("mysql_timeout", 5*time.Minute, "how long to wait for mysqld startup")
+	initDBSQLFile = flag.String("init_db_sql_file", "", "path to .sql file to run after mysql_install_db")
 )
 
 func main() {
 	defer exit.Recover()
+	defer logutil.Flush()
 
 	dbconfigs.RegisterFlags(dbconfigs.All...)
 	mysqlctl.RegisterFlags()
@@ -94,42 +107,61 @@ func main() {
 	ctx, cancel := context.WithTimeout(context.Background(), *timeout)
 	defer cancel()
 
-	tabletAlias, err := topoproto.ParseTabletAlias(*tabletPath)
+	// This is an imaginary tablet alias. The value doesn't matter for anything,
+	// except that we generate a random UID to ensure the target backup
+	// directory is unique if multiple vtbackup instances are launched for the
+	// same shard, at exactly the same second, pointed at the same backup
+	// storage location.
+	bigN, err := rand.Int(rand.Reader, big.NewInt(math.MaxUint32))
 	if err != nil {
-		log.Errorf("failed to parse -tablet-path: %v", err)
+		log.Errorf("can't generate random tablet UID: %v", err)
 		exit.Return(1)
 	}
+	tabletAlias := &topodatapb.TabletAlias{
+		Cell: "vtbackup",
+		Uid:  uint32(bigN.Uint64()),
+	}
+
+	// Clean up our temporary data dir if we exit for any reason, to make sure
+	// every invocation of vtbackup starts with a clean slate, and it does not
+	// accumulate garbage (and run out of disk space) if it's restarted.
+	tabletDir := mysqlctl.TabletDir(tabletAlias.Uid)
+	defer func() {
+		log.Infof("Removing temporary tablet directory: %v", tabletDir)
+		if err := os.RemoveAll(tabletDir); err != nil {
+			log.Errorf("Failed to remove temporary tablet directory: %v", err)
+		}
+	}()
 
+	// Start up mysqld as if we are mysqlctld provisioning a fresh tablet.
+	mysqld, mycnf, err := mysqlctl.CreateMysqldAndMycnf(tabletAlias.Uid, *mysqlSocket, int32(*mysqlPort))
+	if err != nil {
+		log.Errorf("failed to initialize mysql config: %v", err)
+		exit.Return(1)
+	}
+	initCtx, initCancel := context.WithTimeout(ctx, *mysqlTimeout)
+	defer initCancel()
+	if err := mysqld.Init(initCtx, mycnf, *initDBSQLFile); err != nil {
+		log.Errorf("failed to initialize mysql data dir and start mysqld: %v", err)
+		exit.Return(1)
+	}
+	// Shut down mysqld when we're done.
+	defer func() {
+		// Be careful not to use the original context, because we don't want to
+		// skip shutdown just because we timed out waiting for other things.
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		mysqld.Shutdown(ctx, mycnf, false)
+	}()
+
+	// Restore from backup.
 	dbName := *initDbNameOverride
 	if dbName == "" {
 		dbName = fmt.Sprintf("vt_%s", *initKeyspace)
 	}
-
-	var mycnf *mysqlctl.Mycnf
-	var socketFile string
 	extraEnv := map[string]string{
 		"TABLET_ALIAS": topoproto.TabletAliasString(tabletAlias),
 	}
-
-	if dbconfigs.HasConnectionParams() {
-		log.Info("connection parameters were specified. Not loading my.cnf.")
-	} else {
-		var err error
-		if mycnf, err = mysqlctl.NewMycnfFromFlags(tabletAlias.Uid); err != nil {
-			log.Errorf("mycnf read failed: %v", err)
-			exit.Return(1)
-		}
-		socketFile = mycnf.SocketFile
-	}
-
-	dbcfgs, err := dbconfigs.Init(socketFile)
-	if err != nil {
-		log.Errorf("can't initialize dbconfigs: %v", err)
-		exit.Return(1)
-	}
-
-	topoServer := topo.Open()
-	mysqld := mysqlctl.NewMysqld(dbcfgs)
 	dir := fmt.Sprintf("%v/%v", *initKeyspace, *initShard)
 
 	log.Infof("Restoring latest backup from directory %v", dir)
@@ -153,6 +185,8 @@ func main() {
 		log.Errorf("Error resetting replication %v", err)
 		exit.Return(1)
 	}
+	topoServer := topo.Open()
+	defer topoServer.Close()
 	if err := startReplication(ctx, pos, mysqld, topoServer); err != nil {
 		log.Errorf("Error starting replication %v", err)
 		exit.Return(1)

From e89e429c28802187f981da3a7c0cd85b8ff2ec42 Mon Sep 17 00:00:00 2001
From: Anthony Yeh <enisoc@planetscale.com>
Date: Tue, 4 Jun 2019 17:35:40 -0700
Subject: [PATCH 3/8] vtbackup: Add to vitess/lite docker images.

Signed-off-by: Anthony Yeh <enisoc@planetscale.com>
---
 docker/lite/Dockerfile            | 3 ++-
 docker/lite/Dockerfile.alpine     | 1 +
 docker/lite/Dockerfile.mariadb    | 1 +
 docker/lite/Dockerfile.mariadb103 | 1 +
 docker/lite/Dockerfile.mysql56    | 3 ++-
 docker/lite/Dockerfile.mysql57    | 3 ++-
 docker/lite/Dockerfile.mysql80    | 3 ++-
 docker/lite/Dockerfile.percona    | 1 +
 docker/lite/Dockerfile.percona57  | 1 +
 docker/lite/Dockerfile.percona80  | 1 +
 10 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/docker/lite/Dockerfile b/docker/lite/Dockerfile
index 53e0006653d..78318e5c652 100644
--- a/docker/lite/Dockerfile
+++ b/docker/lite/Dockerfile
@@ -15,6 +15,7 @@ COPY --from=builder /vt/bin/vtctlclient /vt/bin/
 COPY --from=builder /vt/bin/vtgate /vt/bin/
 COPY --from=builder /vt/bin/vttablet /vt/bin/
 COPY --from=builder /vt/bin/vtworker /vt/bin/
+COPY --from=builder /vt/bin/vtbackup /vt/bin/
 
 RUN chown -R vitess:vitess /vt
 
@@ -51,4 +52,4 @@ COPY --from=staging /vt/ /vt/
 
 # Create mount point for actual data (e.g. MySQL data dir)
 VOLUME /vt/vtdataroot
-USER vitess
\ No newline at end of file
+USER vitess
diff --git a/docker/lite/Dockerfile.alpine b/docker/lite/Dockerfile.alpine
index 6bc9ccf0492..f3d282163a9 100644
--- a/docker/lite/Dockerfile.alpine
+++ b/docker/lite/Dockerfile.alpine
@@ -13,6 +13,7 @@ COPY --from=builder /vt/bin/vtctlclient /vt/bin/
 COPY --from=builder /vt/bin/vtgate /vt/bin/
 COPY --from=builder /vt/bin/vttablet /vt/bin/
 COPY --from=builder /vt/bin/vtworker /vt/bin/
+COPY --from=builder /vt/bin/vtbackup /vt/bin/
 
 FROM alpine:3.8 
 
diff --git a/docker/lite/Dockerfile.mariadb b/docker/lite/Dockerfile.mariadb
index 8684e2b59f8..cea94d615e5 100644
--- a/docker/lite/Dockerfile.mariadb
+++ b/docker/lite/Dockerfile.mariadb
@@ -15,6 +15,7 @@ COPY --from=builder /vt/bin/vtctlclient /vt/bin/
 COPY --from=builder /vt/bin/vtgate /vt/bin/
 COPY --from=builder /vt/bin/vttablet /vt/bin/
 COPY --from=builder /vt/bin/vtworker /vt/bin/
+COPY --from=builder /vt/bin/vtbackup /vt/bin/
 
 RUN chown -R vitess:vitess /vt
 
diff --git a/docker/lite/Dockerfile.mariadb103 b/docker/lite/Dockerfile.mariadb103
index fd6e9f48d88..4ff440d3d86 100644
--- a/docker/lite/Dockerfile.mariadb103
+++ b/docker/lite/Dockerfile.mariadb103
@@ -15,6 +15,7 @@ COPY --from=builder /vt/bin/vtctlclient /vt/bin/
 COPY --from=builder /vt/bin/vtgate /vt/bin/
 COPY --from=builder /vt/bin/vttablet /vt/bin/
 COPY --from=builder /vt/bin/vtworker /vt/bin/
+COPY --from=builder /vt/bin/vtbackup /vt/bin/
 RUN chown -R vitess:vitess /vt
 
 FROM debian:stretch-slim
diff --git a/docker/lite/Dockerfile.mysql56 b/docker/lite/Dockerfile.mysql56
index f0bb24ecc3b..f3d6b3dcb7a 100644
--- a/docker/lite/Dockerfile.mysql56
+++ b/docker/lite/Dockerfile.mysql56
@@ -15,6 +15,7 @@ COPY --from=builder /vt/bin/vtctlclient /vt/bin/
 COPY --from=builder /vt/bin/vtgate /vt/bin/
 COPY --from=builder /vt/bin/vttablet /vt/bin/
 COPY --from=builder /vt/bin/vtworker /vt/bin/
+COPY --from=builder /vt/bin/vtbackup /vt/bin/
 
 RUN chown -R vitess:vitess /vt
 
@@ -51,4 +52,4 @@ COPY --from=staging /vt/ /vt/
 
 # Create mount point for actual data (e.g. MySQL data dir)
 VOLUME /vt/vtdataroot
-USER vitess
\ No newline at end of file
+USER vitess
diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57
index 53e0006653d..78318e5c652 100644
--- a/docker/lite/Dockerfile.mysql57
+++ b/docker/lite/Dockerfile.mysql57
@@ -15,6 +15,7 @@ COPY --from=builder /vt/bin/vtctlclient /vt/bin/
 COPY --from=builder /vt/bin/vtgate /vt/bin/
 COPY --from=builder /vt/bin/vttablet /vt/bin/
 COPY --from=builder /vt/bin/vtworker /vt/bin/
+COPY --from=builder /vt/bin/vtbackup /vt/bin/
 
 RUN chown -R vitess:vitess /vt
 
@@ -51,4 +52,4 @@ COPY --from=staging /vt/ /vt/
 
 # Create mount point for actual data (e.g. MySQL data dir)
 VOLUME /vt/vtdataroot
-USER vitess
\ No newline at end of file
+USER vitess
diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80
index 3d20b60dee5..fb71b6f6b56 100644
--- a/docker/lite/Dockerfile.mysql80
+++ b/docker/lite/Dockerfile.mysql80
@@ -15,6 +15,7 @@ COPY --from=builder /vt/bin/vtctlclient /vt/bin/
 COPY --from=builder /vt/bin/vtgate /vt/bin/
 COPY --from=builder /vt/bin/vttablet /vt/bin/
 COPY --from=builder /vt/bin/vtworker /vt/bin/
+COPY --from=builder /vt/bin/vtbackup /vt/bin/
 
 RUN chown -R vitess:vitess /vt
 
@@ -51,4 +52,4 @@ COPY --from=staging /vt/ /vt/
 
 # Create mount point for actual data (e.g. MySQL data dir)
 VOLUME /vt/vtdataroot
-USER vitess
\ No newline at end of file
+USER vitess
diff --git a/docker/lite/Dockerfile.percona b/docker/lite/Dockerfile.percona
index bf3b06a69d8..e8d127dc56f 100644
--- a/docker/lite/Dockerfile.percona
+++ b/docker/lite/Dockerfile.percona
@@ -15,6 +15,7 @@ COPY --from=builder /vt/bin/vtctlclient /vt/bin/
 COPY --from=builder /vt/bin/vtgate /vt/bin/
 COPY --from=builder /vt/bin/vttablet /vt/bin/
 COPY --from=builder /vt/bin/vtworker /vt/bin/
+COPY --from=builder /vt/bin/vtbackup /vt/bin/
 
 RUN chown -R vitess:vitess /vt
 
diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57
index 4b1a55b478d..30472af2a60 100644
--- a/docker/lite/Dockerfile.percona57
+++ b/docker/lite/Dockerfile.percona57
@@ -15,6 +15,7 @@ COPY --from=builder /vt/bin/vtctlclient /vt/bin/
 COPY --from=builder /vt/bin/vtgate /vt/bin/
 COPY --from=builder /vt/bin/vttablet /vt/bin/
 COPY --from=builder /vt/bin/vtworker /vt/bin/
+COPY --from=builder /vt/bin/vtbackup /vt/bin/
 
 RUN chown -R vitess:vitess /vt
 
diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80
index ceec322e36c..31182d277d6 100644
--- a/docker/lite/Dockerfile.percona80
+++ b/docker/lite/Dockerfile.percona80
@@ -15,6 +15,7 @@ COPY --from=builder /vt/bin/vtctlclient /vt/bin/
 COPY --from=builder /vt/bin/vtgate /vt/bin/
 COPY --from=builder /vt/bin/vttablet /vt/bin/
 COPY --from=builder /vt/bin/vtworker /vt/bin/
+COPY --from=builder /vt/bin/vtbackup /vt/bin/
 
 RUN chown -R vitess:vitess /vt
 

From b42cfb25bbaffb4133892cb1624978a54c6a92bb Mon Sep 17 00:00:00 2001
From: Anthony Yeh <enisoc@planetscale.com>
Date: Tue, 4 Jun 2019 19:37:21 -0700
Subject: [PATCH 4/8] vtbackup: Use a snapshot position as the goal instead of
 a lag value.

The lag value can lie. Also we're guaranteed to hit the snapshot
position eventually as long as replication is making progress. If our
goal is a lag value, we may never catch up if replication is slower than
the rate of new transactions on the master.

Signed-off-by: Anthony Yeh <enisoc@planetscale.com>
---
 go/cmd/vtbackup/vtbackup.go | 126 +++++++++++++++++++++++++++++-------
 1 file changed, 101 insertions(+), 25 deletions(-)

diff --git a/go/cmd/vtbackup/vtbackup.go b/go/cmd/vtbackup/vtbackup.go
index e8ce0c499c3..c968692c006 100644
--- a/go/cmd/vtbackup/vtbackup.go
+++ b/go/cmd/vtbackup/vtbackup.go
@@ -37,8 +37,11 @@ The process vtbackup follows to take a new backup is as follows:
 2. Start a mysqld instance (but no vttablet) from the restored data.
 3. Instruct mysqld to connect to the current shard master and replicate any
    transactions that are new since the last backup.
-4. Wait until replication is caught up to the master.
-5. Stop mysqld and take a new backup.
+4. Ask the master for its current replication position and set that as the goal
+   for catching up on replication before taking the backup, so the goalposts
+   don't move.
+5. Wait until replication is caught up to the goal position or beyond.
+6. Stop mysqld and take a new backup.
 
 Aside from additional replication load while vtbackup's mysqld catches up on
 new transactions, the shard should be otherwise unaffected. Existing tablets
@@ -75,12 +78,14 @@ import (
 	"vitess.io/vitess/go/vt/topo"
 	"vitess.io/vitess/go/vt/topo/topoproto"
 	"vitess.io/vitess/go/vt/vterrors"
+	_ "vitess.io/vitess/go/vt/vttablet/grpctmclient"
+	"vitess.io/vitess/go/vt/vttablet/tmclient"
 )
 
 var (
 	// vtbackup-specific flags
-	acceptableReplicationLag = flag.Duration("acceptable_replication_lag", 1*time.Second, "Wait until replication lag is less than or equal to this value before taking a new backup")
-	timeout                  = flag.Duration("timeout", 1*time.Hour, "Overall timeout for this vtbackup run")
+	timeout            = flag.Duration("timeout", 2*time.Hour, "Overall timeout for this whole vtbackup run, including restoring the previous backup, waiting for replication, and uploading files")
+	replicationTimeout = flag.Duration("replication_timeout", 1*time.Hour, "The timeout for the step of waiting for replication to catch up. If progress is made before this timeout is reached, the backup will be taken anyway to save partial progress, but vtbackup will return a non-zero exit code to indicate it should be retried since not all expected data was backed up")
 
 	// vttablet-like flags
 	initDbNameOverride = flag.String("init_db_name_override", "", "(init parameter) override the name of the db used by vttablet")
@@ -165,10 +170,10 @@ func main() {
 	dir := fmt.Sprintf("%v/%v", *initKeyspace, *initShard)
 
 	log.Infof("Restoring latest backup from directory %v", dir)
-	pos, err := mysqlctl.Restore(ctx, mycnf, mysqld, dir, *concurrency, extraEnv, map[string]string{}, logutil.NewConsoleLogger(), true, dbName)
+	restorePos, err := mysqlctl.Restore(ctx, mycnf, mysqld, dir, *concurrency, extraEnv, map[string]string{}, logutil.NewConsoleLogger(), true, dbName)
 	switch err {
 	case nil:
-		log.Info("Successfully restored from backup at replication position %v", pos)
+		log.Infof("Successfully restored from backup at replication position %v", restorePos)
 	case mysqlctl.ErrNoBackup:
 		log.Error("No backup found. Not starting up empty since -initial_backup flag was not enabled.")
 		exit.Return(1)
@@ -181,26 +186,53 @@ func main() {
 	}
 
 	// We have restored a backup. Now start replication.
-	if err := resetReplication(ctx, pos, mysqld); err != nil {
+	if err := resetReplication(ctx, restorePos, mysqld); err != nil {
 		log.Errorf("Error resetting replication %v", err)
 		exit.Return(1)
 	}
 	topoServer := topo.Open()
 	defer topoServer.Close()
-	if err := startReplication(ctx, pos, mysqld, topoServer); err != nil {
+	if err := startReplication(ctx, mysqld, topoServer); err != nil {
 		log.Errorf("Error starting replication %v", err)
 		exit.Return(1)
 	}
 
+	// Get the current master replication position, and wait until we catch up
+	// to that point. We do this instead of looking at Seconds_Behind_Master
+	// (replication lag reported by SHOW SLAVE STATUS) because that value can
+	// sometimes lie and tell you there's 0 lag when actually replication is
+	// stopped. Also, if replication is making progress but is too slow to ever
+	// catch up to live changes, we'd rather take a backup of something rather
+	// than timing out.
+	tmc := tmclient.NewTabletManagerClient()
+	masterPos, err := getMasterPosition(ctx, tmc, topoServer)
+	if err != nil {
+		log.Errorf("Can't get the master replication position: %v", err)
+		exit.Return(1)
+	}
+
+	// Remember the time when we fetched the master position, not when we caught
+	// up to it, so the timestamp on our backup is honest (assuming we make it
+	// to the goal position).
+	backupTime := time.Now()
+
+	if restorePos.Equal(masterPos) {
+		// Nothing has happened on the master since the last backup, so there's
+		// no point taking a new backup since it would be identical.
+		log.Infof("No backup is necessary. The latest backup is up-to-date with the master.")
+		return
+	}
+
 	// Wait for replication to catch up.
 	waitStartTime := time.Now()
 	for {
 		time.Sleep(time.Second)
 
-		// Check if the context is still good.
-		if err := ctx.Err(); err != nil {
-			log.Errorf("Timed out waiting for replication to catch up to within %v.", *acceptableReplicationLag)
-			exit.Return(1)
+		// Check if the replication context is still good.
+		if time.Since(waitStartTime) > *replicationTimeout {
+			// If we time out on this step, we still might take the backup anyway.
+			log.Errorf("Timed out waiting for replication to catch up to %v.", masterPos)
+			break
 		}
 
 		status, statusErr := mysqld.SlaveStatus()
@@ -208,25 +240,45 @@ func main() {
 			log.Warningf("Error getting replication status: %v", statusErr)
 			continue
 		}
-		if time.Duration(status.SecondsBehindMaster)*time.Second <= *acceptableReplicationLag {
-			// We're caught up on replication.
-			log.Infof("Replication caught up to within %v after %v", *acceptableReplicationLag, time.Since(waitStartTime))
+		if status.Position.AtLeast(masterPos) {
+			// We're caught up on replication to at least the point the master
+			// was at when this vtbackup run started.
+			log.Infof("Replication caught up to %v after %v", status.Position, time.Since(waitStartTime))
 			break
 		}
 		if !status.SlaveRunning() {
 			log.Warning("Replication has stopped before backup could be taken. Trying to restart replication.")
-			if err := startReplication(ctx, pos, mysqld, topoServer); err != nil {
+			if err := startReplication(ctx, mysqld, topoServer); err != nil {
 				log.Warningf("Failed to restart replication: %v", err)
 			}
 		}
 	}
 
+	// Did we make any progress?
+	status, err := mysqld.SlaveStatus()
+	if err != nil {
+		log.Errorf("Error getting replication status: %v", err)
+		exit.Return(1)
+	}
+	log.Infof("Replication caught up to at least %v", status.Position)
+	if status.Position.Equal(restorePos) {
+		log.Errorf("Not taking backup: replication did not make any progress from restore point: %v", restorePos)
+		exit.Return(1)
+	}
+
 	// Now we can take a new backup.
-	name := fmt.Sprintf("%v.%v", time.Now().UTC().Format("2006-01-02.150405"), topoproto.TabletAliasString(tabletAlias))
+	name := fmt.Sprintf("%v.%v", backupTime.UTC().Format("2006-01-02.150405"), topoproto.TabletAliasString(tabletAlias))
 	if err := mysqlctl.Backup(ctx, mycnf, mysqld, logutil.NewConsoleLogger(), dir, name, *concurrency, extraEnv); err != nil {
 		log.Errorf("Error taking backup: %v", err)
 		exit.Return(1)
 	}
+
+	// Return a non-zero exit code if we didn't meet the replication position
+	// goal, even though we took a backup that pushes the high-water mark up.
+	if !status.Position.AtLeast(masterPos) {
+		log.Warningf("Replication caught up to %v but didn't make it to the goal of %v. A backup was taken anyway to save partial progress, but the operation should still be retried since not all expected data is backed up.", status.Position, masterPos)
+		exit.Return(1)
+	}
 }
 
 func resetReplication(ctx context.Context, pos mysql.Position, mysqld mysqlctl.MysqlDaemon) error {
@@ -245,19 +297,18 @@ func resetReplication(ctx context.Context, pos mysql.Position, mysqld mysqlctl.M
 	return nil
 }
 
-func startReplication(ctx context.Context, pos mysql.Position, mysqld mysqlctl.MysqlDaemon, topoServer *topo.Server) error {
+func startReplication(ctx context.Context, mysqld mysqlctl.MysqlDaemon, topoServer *topo.Server) error {
 	si, err := topoServer.GetShard(ctx, *initKeyspace, *initShard)
 	if err != nil {
 		return vterrors.Wrap(err, "can't read shard")
 	}
-	if si.MasterAlias == nil {
-		// We've restored, but there's no master. This is fine, since we've
-		// already set the position at which to resume when we're later reparented.
-		// If we had instead considered this fatal, all tablets would crash-loop
-		// until a master appears, which would make it impossible to elect a master.
-		log.Warning("Can't start replication after restore: shard has no master.")
-		return nil
+	if topoproto.TabletAliasIsZero(si.MasterAlias) {
+		// Normal tablets will sit around waiting to be reparented in this case.
+		// Since vtbackup is a batch job, we just have to fail.
+		return fmt.Errorf("can't start replication after restore: shard %v/%v has no master", *initKeyspace, *initShard)
 	}
+	// TODO(enisoc): Support replicating from another replica, preferably in the
+	//   same cell, preferably rdonly, to reduce load on the master.
 	ti, err := topoServer.GetTablet(ctx, si.MasterAlias)
 	if err != nil {
 		return vterrors.Wrapf(err, "Cannot read master tablet %v", si.MasterAlias)
@@ -269,3 +320,28 @@ func startReplication(ctx context.Context, pos mysql.Position, mysqld mysqlctl.M
 	}
 	return nil
 }
+
+func getMasterPosition(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server) (mysql.Position, error) {
+	si, err := ts.GetShard(ctx, *initKeyspace, *initShard)
+	if err != nil {
+		return mysql.Position{}, vterrors.Wrap(err, "can't read shard")
+	}
+	if topoproto.TabletAliasIsZero(si.MasterAlias) {
+		// Normal tablets will sit around waiting to be reparented in this case.
+		// Since vtbackup is a batch job, we just have to fail.
+		return mysql.Position{}, fmt.Errorf("shard %v/%v has no master", *initKeyspace, *initShard)
+	}
+	ti, err := ts.GetTablet(ctx, si.MasterAlias)
+	if err != nil {
+		return mysql.Position{}, fmt.Errorf("can't get master tablet record %v: %v", topoproto.TabletAliasString(si.MasterAlias), err)
+	}
+	posStr, err := tmc.MasterPosition(ctx, ti.Tablet)
+	if err != nil {
+		return mysql.Position{}, fmt.Errorf("can't get master replication position: %v", err)
+	}
+	pos, err := mysql.DecodePosition(posStr)
+	if err != nil {
+		return mysql.Position{}, fmt.Errorf("can't decode master replication position %q: %v", posStr, err)
+	}
+	return pos, nil
+}

From e10c6246d8543eeda0d42a8a3ec8cd3ceab8e080 Mon Sep 17 00:00:00 2001
From: Anthony Yeh <enisoc@planetscale.com>
Date: Tue, 4 Jun 2019 21:21:51 -0700
Subject: [PATCH 5/8] vtbackup: Add -initial_backup flag for seeding an empty
 backup.

Signed-off-by: Anthony Yeh <enisoc@planetscale.com>
---
 go/cmd/vtbackup/vtbackup.go | 67 +++++++++++++++++++++++++++++++++----
 1 file changed, 60 insertions(+), 7 deletions(-)

diff --git a/go/cmd/vtbackup/vtbackup.go b/go/cmd/vtbackup/vtbackup.go
index c968692c006..f2ac4e03133 100644
--- a/go/cmd/vtbackup/vtbackup.go
+++ b/go/cmd/vtbackup/vtbackup.go
@@ -73,6 +73,7 @@ import (
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/logutil"
 	"vitess.io/vitess/go/vt/mysqlctl"
+	"vitess.io/vitess/go/vt/mysqlctl/backupstorage"
 	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/servenv"
 	"vitess.io/vitess/go/vt/topo"
@@ -86,6 +87,7 @@ var (
 	// vtbackup-specific flags
 	timeout            = flag.Duration("timeout", 2*time.Hour, "Overall timeout for this whole vtbackup run, including restoring the previous backup, waiting for replication, and uploading files")
 	replicationTimeout = flag.Duration("replication_timeout", 1*time.Hour, "The timeout for the step of waiting for replication to catch up. If progress is made before this timeout is reached, the backup will be taken anyway to save partial progress, but vtbackup will return a non-zero exit code to indicate it should be retried since not all expected data was backed up")
+	initialBackup      = flag.Bool("initial_backup", false, "Instead of restoring from backup, initialize an empty database with the provided init_db_sql_file and upload a backup of that for the shard. This can be used to seed a brand new shard with an initial, empty backup. This can only be done before the shard exists in topology (i.e. before any tablets are deployed), and before any backups exist for the shard")
 
 	// vttablet-like flags
 	initDbNameOverride = flag.String("init_db_name_override", "", "(init parameter) override the name of the db used by vttablet")
@@ -159,15 +161,63 @@ func main() {
 		mysqld.Shutdown(ctx, mycnf, false)
 	}()
 
+	extraEnv := map[string]string{
+		"TABLET_ALIAS": topoproto.TabletAliasString(tabletAlias),
+	}
+	dir := fmt.Sprintf("%v/%v", *initKeyspace, *initShard)
+	topoServer := topo.Open()
+	defer topoServer.Close()
+
+	// In initial_backup mode, just take a backup of this empty database.
+	if *initialBackup {
+		// Check that the shard doesn't exist.
+		_, err := topoServer.GetShard(ctx, *initKeyspace, *initShard)
+		if !topo.IsErrType(err, topo.NoNode) {
+			log.Errorf("Refusing to upload initial backup of empty database: the shard %v/%v already exists in topology.", *initKeyspace, *initShard)
+			exit.Return(1)
+		}
+		// Check that no existing backups exist in this backup storage location.
+		bs, err := backupstorage.GetBackupStorage()
+		if err != nil {
+			log.Errorf("Can't get backup storage: %v", err)
+			exit.Return(1)
+		}
+		backups, err := bs.ListBackups(ctx, dir)
+		if err != nil {
+			log.Errorf("Can't list backups: %v", err)
+			exit.Return(1)
+		}
+		if len(backups) > 0 {
+			log.Errorf("Refusing to upload initial backup of empty database: the shard %v/%v already has at least one backup.", *initKeyspace, *initShard)
+			exit.Return(1)
+		}
+
+		// If the checks pass, go ahead and take a backup of this empty DB.
+		// First, initialize it the way InitShardMaster would, so this backup
+		// produces a result that can be used to skip InitShardMaster entirely.
+		// This involves resetting replication (to erase any history) and then
+		// executing a statement to force the creation of a replication position.
+		mysqld.ResetReplication(ctx)
+		cmds := mysqlctl.CreateReparentJournal()
+		if err := mysqld.ExecuteSuperQueryList(ctx, cmds); err != nil {
+			log.Errorf("Can't initialize database with reparent journal: %v", err)
+			exit.Return(1)
+		}
+		// Now we're ready to take the backup.
+		name := backupName(time.Now(), tabletAlias)
+		if err := mysqlctl.Backup(ctx, mycnf, mysqld, logutil.NewConsoleLogger(), dir, name, *concurrency, extraEnv); err != nil {
+			log.Errorf("Error taking backup: %v", err)
+			exit.Return(1)
+		}
+		log.Info("Backup successful")
+		return
+	}
+
 	// Restore from backup.
 	dbName := *initDbNameOverride
 	if dbName == "" {
 		dbName = fmt.Sprintf("vt_%s", *initKeyspace)
 	}
-	extraEnv := map[string]string{
-		"TABLET_ALIAS": topoproto.TabletAliasString(tabletAlias),
-	}
-	dir := fmt.Sprintf("%v/%v", *initKeyspace, *initShard)
 
 	log.Infof("Restoring latest backup from directory %v", dir)
 	restorePos, err := mysqlctl.Restore(ctx, mycnf, mysqld, dir, *concurrency, extraEnv, map[string]string{}, logutil.NewConsoleLogger(), true, dbName)
@@ -190,8 +240,6 @@ func main() {
 		log.Errorf("Error resetting replication %v", err)
 		exit.Return(1)
 	}
-	topoServer := topo.Open()
-	defer topoServer.Close()
 	if err := startReplication(ctx, mysqld, topoServer); err != nil {
 		log.Errorf("Error starting replication %v", err)
 		exit.Return(1)
@@ -267,7 +315,7 @@ func main() {
 	}
 
 	// Now we can take a new backup.
-	name := fmt.Sprintf("%v.%v", backupTime.UTC().Format("2006-01-02.150405"), topoproto.TabletAliasString(tabletAlias))
+	name := backupName(backupTime, tabletAlias)
 	if err := mysqlctl.Backup(ctx, mycnf, mysqld, logutil.NewConsoleLogger(), dir, name, *concurrency, extraEnv); err != nil {
 		log.Errorf("Error taking backup: %v", err)
 		exit.Return(1)
@@ -279,6 +327,7 @@ func main() {
 		log.Warningf("Replication caught up to %v but didn't make it to the goal of %v. A backup was taken anyway to save partial progress, but the operation should still be retried since not all expected data is backed up.", status.Position, masterPos)
 		exit.Return(1)
 	}
+	log.Info("Backup successful")
 }
 
 func resetReplication(ctx context.Context, pos mysql.Position, mysqld mysqlctl.MysqlDaemon) error {
@@ -345,3 +394,7 @@ func getMasterPosition(ctx context.Context, tmc tmclient.TabletManagerClient, ts
 	}
 	return pos, nil
 }
+
+func backupName(backupTime time.Time, tabletAlias *topodatapb.TabletAlias) string {
+	return fmt.Sprintf("%v.%v", backupTime.UTC().Format("2006-01-02.150405"), topoproto.TabletAliasString(tabletAlias))
+}

From 22bccdd3d02a2912b71ad54c11ec7c8fde14fb98 Mon Sep 17 00:00:00 2001
From: Anthony Yeh <enisoc@planetscale.com>
Date: Wed, 5 Jun 2019 09:53:53 -0700
Subject: [PATCH 6/8] vtbackup: Add backup interval and pruning with retention
 policy.

Signed-off-by: Anthony Yeh <enisoc@planetscale.com>
---
 go/cmd/vtbackup/vtbackup.go | 244 ++++++++++++++++++++++++++----------
 1 file changed, 177 insertions(+), 67 deletions(-)

diff --git a/go/cmd/vtbackup/vtbackup.go b/go/cmd/vtbackup/vtbackup.go
index f2ac4e03133..b3e5d05f77c 100644
--- a/go/cmd/vtbackup/vtbackup.go
+++ b/go/cmd/vtbackup/vtbackup.go
@@ -65,6 +65,7 @@ import (
 	"math"
 	"math/big"
 	"os"
+	"strings"
 	"time"
 
 	"vitess.io/vitess/go/exit"
@@ -83,11 +84,20 @@ import (
 	"vitess.io/vitess/go/vt/vttablet/tmclient"
 )
 
+const (
+	backupTimestampFormat = "2006-01-02.150405"
+)
+
 var (
 	// vtbackup-specific flags
 	timeout            = flag.Duration("timeout", 2*time.Hour, "Overall timeout for this whole vtbackup run, including restoring the previous backup, waiting for replication, and uploading files")
 	replicationTimeout = flag.Duration("replication_timeout", 1*time.Hour, "The timeout for the step of waiting for replication to catch up. If progress is made before this timeout is reached, the backup will be taken anyway to save partial progress, but vtbackup will return a non-zero exit code to indicate it should be retried since not all expected data was backed up")
-	initialBackup      = flag.Bool("initial_backup", false, "Instead of restoring from backup, initialize an empty database with the provided init_db_sql_file and upload a backup of that for the shard. This can be used to seed a brand new shard with an initial, empty backup. This can only be done before the shard exists in topology (i.e. before any tablets are deployed), and before any backups exist for the shard")
+
+	minBackupInterval = flag.Duration("min_backup_interval", 0, "Only take a new backup if it's been at least this long since the most recent backup.")
+	minRetentionTime  = flag.Duration("min_retention_time", 0, "Keep each old backup for at least this long before removing it. Set to 0 to disable pruning of old backups.")
+	minRetentionCount = flag.Int("min_retention_count", 1, "Always keep at least this many of the most recent backups in this backup storage location, even if some are older than the min_retention_time. This must be at least 1 since a backup must always exist to allow new backups to be made")
+
+	initialBackup = flag.Bool("initial_backup", false, "Instead of restoring from backup, initialize an empty database with the provided init_db_sql_file and upload a backup of that for the shard. This can be used to seed a brand new shard with an initial, empty backup. This can only be done before the shard exists in topology (i.e. before any tablets are deployed), and before any backups exist for the shard")
 
 	// vttablet-like flags
 	initDbNameOverride = flag.String("init_db_name_override", "", "(init parameter) override the name of the db used by vttablet")
@@ -111,9 +121,49 @@ func main() {
 
 	servenv.ParseFlags("vtbackup")
 
+	if *minRetentionCount < 1 {
+		log.Errorf("min_retention_count must be at least 1 to allow restores to succeed")
+		exit.Return(1)
+	}
+
 	ctx, cancel := context.WithTimeout(context.Background(), *timeout)
 	defer cancel()
 
+	// Open connection backup storage.
+	backupDir := fmt.Sprintf("%v/%v", *initKeyspace, *initShard)
+	backupStorage, err := backupstorage.GetBackupStorage()
+	if err != nil {
+		log.Errorf("Can't get backup storage: %v", err)
+		exit.Return(1)
+	}
+	defer backupStorage.Close()
+	// Open connection to topology server.
+	topoServer := topo.Open()
+	defer topoServer.Close()
+
+	// Try to take a backup, if it's been long enough since the last one.
+	// Skip pruning if backup wasn't fully successful. We don't want to be
+	// deleting things if the backup process is not healthy.
+	doBackup, err := shouldBackup(ctx, topoServer, backupStorage, backupDir)
+	if err != nil {
+		log.Errorf("Can't take backup: %v", err)
+		exit.Return(1)
+	}
+	if doBackup {
+		if err := takeBackup(ctx, topoServer, backupStorage, backupDir); err != nil {
+			log.Errorf("Failed to take backup: %v", err)
+			exit.Return(1)
+		}
+	}
+
+	// Prune old backups.
+	if err := pruneBackups(ctx, backupStorage, backupDir); err != nil {
+		log.Errorf("Couldn't prune old backups: %v", err)
+		exit.Return(1)
+	}
+}
+
+func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage backupstorage.BackupStorage, backupDir string) error {
 	// This is an imaginary tablet alias. The value doesn't matter for anything,
 	// except that we generate a random UID to ensure the target backup
 	// directory is unique if multiple vtbackup instances are launched for the
@@ -121,8 +171,7 @@ func main() {
 	// storage location.
 	bigN, err := rand.Int(rand.Reader, big.NewInt(math.MaxUint32))
 	if err != nil {
-		log.Errorf("can't generate random tablet UID: %v", err)
-		exit.Return(1)
+		return fmt.Errorf("can't generate random tablet UID: %v", err)
 	}
 	tabletAlias := &topodatapb.TabletAlias{
 		Cell: "vtbackup",
@@ -136,21 +185,19 @@ func main() {
 	defer func() {
 		log.Infof("Removing temporary tablet directory: %v", tabletDir)
 		if err := os.RemoveAll(tabletDir); err != nil {
-			log.Errorf("Failed to remove temporary tablet directory: %v", err)
+			log.Warningf("Failed to remove temporary tablet directory: %v", err)
 		}
 	}()
 
 	// Start up mysqld as if we are mysqlctld provisioning a fresh tablet.
 	mysqld, mycnf, err := mysqlctl.CreateMysqldAndMycnf(tabletAlias.Uid, *mysqlSocket, int32(*mysqlPort))
 	if err != nil {
-		log.Errorf("failed to initialize mysql config: %v", err)
-		exit.Return(1)
+		return fmt.Errorf("failed to initialize mysql config: %v", err)
 	}
 	initCtx, initCancel := context.WithTimeout(ctx, *mysqlTimeout)
 	defer initCancel()
 	if err := mysqld.Init(initCtx, mycnf, *initDBSQLFile); err != nil {
-		log.Errorf("failed to initialize mysql data dir and start mysqld: %v", err)
-		exit.Return(1)
+		return fmt.Errorf("failed to initialize mysql data dir and start mysqld: %v", err)
 	}
 	// Shut down mysqld when we're done.
 	defer func() {
@@ -164,35 +211,10 @@ func main() {
 	extraEnv := map[string]string{
 		"TABLET_ALIAS": topoproto.TabletAliasString(tabletAlias),
 	}
-	dir := fmt.Sprintf("%v/%v", *initKeyspace, *initShard)
-	topoServer := topo.Open()
-	defer topoServer.Close()
 
 	// In initial_backup mode, just take a backup of this empty database.
 	if *initialBackup {
-		// Check that the shard doesn't exist.
-		_, err := topoServer.GetShard(ctx, *initKeyspace, *initShard)
-		if !topo.IsErrType(err, topo.NoNode) {
-			log.Errorf("Refusing to upload initial backup of empty database: the shard %v/%v already exists in topology.", *initKeyspace, *initShard)
-			exit.Return(1)
-		}
-		// Check that no existing backups exist in this backup storage location.
-		bs, err := backupstorage.GetBackupStorage()
-		if err != nil {
-			log.Errorf("Can't get backup storage: %v", err)
-			exit.Return(1)
-		}
-		backups, err := bs.ListBackups(ctx, dir)
-		if err != nil {
-			log.Errorf("Can't list backups: %v", err)
-			exit.Return(1)
-		}
-		if len(backups) > 0 {
-			log.Errorf("Refusing to upload initial backup of empty database: the shard %v/%v already has at least one backup.", *initKeyspace, *initShard)
-			exit.Return(1)
-		}
-
-		// If the checks pass, go ahead and take a backup of this empty DB.
+		// Take a backup of this empty DB without restoring anything.
 		// First, initialize it the way InitShardMaster would, so this backup
 		// produces a result that can be used to skip InitShardMaster entirely.
 		// This involves resetting replication (to erase any history) and then
@@ -200,17 +222,15 @@ func main() {
 		mysqld.ResetReplication(ctx)
 		cmds := mysqlctl.CreateReparentJournal()
 		if err := mysqld.ExecuteSuperQueryList(ctx, cmds); err != nil {
-			log.Errorf("Can't initialize database with reparent journal: %v", err)
-			exit.Return(1)
+			return fmt.Errorf("can't initialize database with reparent journal: %v", err)
 		}
 		// Now we're ready to take the backup.
 		name := backupName(time.Now(), tabletAlias)
-		if err := mysqlctl.Backup(ctx, mycnf, mysqld, logutil.NewConsoleLogger(), dir, name, *concurrency, extraEnv); err != nil {
-			log.Errorf("Error taking backup: %v", err)
-			exit.Return(1)
+		if err := mysqlctl.Backup(ctx, mycnf, mysqld, logutil.NewConsoleLogger(), backupDir, name, *concurrency, extraEnv); err != nil {
+			return fmt.Errorf("backup failed: %v", err)
 		}
-		log.Info("Backup successful")
-		return
+		log.Info("Initial backup successful.")
+		return nil
 	}
 
 	// Restore from backup.
@@ -219,30 +239,25 @@ func main() {
 		dbName = fmt.Sprintf("vt_%s", *initKeyspace)
 	}
 
-	log.Infof("Restoring latest backup from directory %v", dir)
-	restorePos, err := mysqlctl.Restore(ctx, mycnf, mysqld, dir, *concurrency, extraEnv, map[string]string{}, logutil.NewConsoleLogger(), true, dbName)
+	log.Infof("Restoring latest backup from directory %v", backupDir)
+	restorePos, err := mysqlctl.Restore(ctx, mycnf, mysqld, backupDir, *concurrency, extraEnv, map[string]string{}, logutil.NewConsoleLogger(), true, dbName)
 	switch err {
 	case nil:
 		log.Infof("Successfully restored from backup at replication position %v", restorePos)
 	case mysqlctl.ErrNoBackup:
-		log.Error("No backup found. Not starting up empty since -initial_backup flag was not enabled.")
-		exit.Return(1)
+		return fmt.Errorf("no backup found; not starting up empty since -initial_backup flag was not enabled")
 	case mysqlctl.ErrExistingDB:
-		log.Error("Can't run vtbackup because data directory is not empty.")
-		exit.Return(1)
+		return fmt.Errorf("can't run vtbackup because data directory is not empty")
 	default:
-		log.Errorf("Error restoring from backup: %v", err)
-		exit.Return(1)
+		return fmt.Errorf("can't restore from backup: %v", err)
 	}
 
 	// We have restored a backup. Now start replication.
 	if err := resetReplication(ctx, restorePos, mysqld); err != nil {
-		log.Errorf("Error resetting replication %v", err)
-		exit.Return(1)
+		return fmt.Errorf("error resetting replication: %v", err)
 	}
 	if err := startReplication(ctx, mysqld, topoServer); err != nil {
-		log.Errorf("Error starting replication %v", err)
-		exit.Return(1)
+		return fmt.Errorf("error starting replication: %v", err)
 	}
 
 	// Get the current master replication position, and wait until we catch up
@@ -255,8 +270,7 @@ func main() {
 	tmc := tmclient.NewTabletManagerClient()
 	masterPos, err := getMasterPosition(ctx, tmc, topoServer)
 	if err != nil {
-		log.Errorf("Can't get the master replication position: %v", err)
-		exit.Return(1)
+		return fmt.Errorf("can't get the master replication position: %v", err)
 	}
 
 	// Remember the time when we fetched the master position, not when we caught
@@ -268,7 +282,7 @@ func main() {
 		// Nothing has happened on the master since the last backup, so there's
 		// no point taking a new backup since it would be identical.
 		log.Infof("No backup is necessary. The latest backup is up-to-date with the master.")
-		return
+		return nil
 	}
 
 	// Wait for replication to catch up.
@@ -305,29 +319,26 @@ func main() {
 	// Did we make any progress?
 	status, err := mysqld.SlaveStatus()
 	if err != nil {
-		log.Errorf("Error getting replication status: %v", err)
-		exit.Return(1)
+		return fmt.Errorf("can't get replication status: %v", err)
 	}
 	log.Infof("Replication caught up to at least %v", status.Position)
 	if status.Position.Equal(restorePos) {
-		log.Errorf("Not taking backup: replication did not make any progress from restore point: %v", restorePos)
-		exit.Return(1)
+		return fmt.Errorf("not taking backup: replication did not make any progress from restore point: %v", restorePos)
 	}
 
 	// Now we can take a new backup.
 	name := backupName(backupTime, tabletAlias)
-	if err := mysqlctl.Backup(ctx, mycnf, mysqld, logutil.NewConsoleLogger(), dir, name, *concurrency, extraEnv); err != nil {
-		log.Errorf("Error taking backup: %v", err)
-		exit.Return(1)
+	if err := mysqlctl.Backup(ctx, mycnf, mysqld, logutil.NewConsoleLogger(), backupDir, name, *concurrency, extraEnv); err != nil {
+		return fmt.Errorf("error taking backup: %v", err)
 	}
 
 	// Return a non-zero exit code if we didn't meet the replication position
 	// goal, even though we took a backup that pushes the high-water mark up.
 	if !status.Position.AtLeast(masterPos) {
-		log.Warningf("Replication caught up to %v but didn't make it to the goal of %v. A backup was taken anyway to save partial progress, but the operation should still be retried since not all expected data is backed up.", status.Position, masterPos)
-		exit.Return(1)
+		return fmt.Errorf("replication caught up to %v but didn't make it to the goal of %v; a backup was taken anyway to save partial progress, but the operation should still be retried since not all expected data is backed up", status.Position, masterPos)
 	}
-	log.Info("Backup successful")
+	log.Info("Backup successful.")
+	return nil
 }
 
 func resetReplication(ctx context.Context, pos mysql.Position, mysqld mysqlctl.MysqlDaemon) error {
@@ -396,5 +407,104 @@ func getMasterPosition(ctx context.Context, tmc tmclient.TabletManagerClient, ts
 }
 
 func backupName(backupTime time.Time, tabletAlias *topodatapb.TabletAlias) string {
-	return fmt.Sprintf("%v.%v", backupTime.UTC().Format("2006-01-02.150405"), topoproto.TabletAliasString(tabletAlias))
+	return fmt.Sprintf("%v.%v", backupTime.UTC().Format(backupTimestampFormat), topoproto.TabletAliasString(tabletAlias))
+}
+
+func pruneBackups(ctx context.Context, backupStorage backupstorage.BackupStorage, backupDir string) error {
+	if *minRetentionTime == 0 {
+		log.Info("Pruning of old backups is disabled.")
+		return nil
+	}
+	backups, err := backupStorage.ListBackups(ctx, backupDir)
+	if err != nil {
+		return fmt.Errorf("can't list backups: %v", err)
+	}
+	numBackups := len(backups)
+	if numBackups <= *minRetentionCount {
+		log.Infof("Found %v backups. Not pruning any since this is within the min_retention_count of %v.", numBackups, *minRetentionCount)
+		return nil
+	}
+	// We have more than the minimum retention count, so we could afford to
+	// prune some. See if any are beyond the minimum retention time.
+	// ListBackups returns them sorted by oldest first.
+	for _, backup := range backups {
+		backupTime, err := parseBackupTime(backup.Name())
+		if err != nil {
+			return err
+		}
+		if time.Since(backupTime) < *minRetentionTime {
+			// The oldest remaining backup is not old enough to prune.
+			log.Infof("Oldest backup taken at %v has not reached min_retention_time of %v. Nothing left to prune.", backupTime, *minRetentionTime)
+			break
+		}
+		// Remove the backup.
+		log.Infof("Removing old backup %v from %v, since it's older than min_retention_time of %v", backup.Name(), backupDir, *minRetentionTime)
+		if err := backupStorage.RemoveBackup(ctx, backupDir, backup.Name()); err != nil {
+			return fmt.Errorf("couldn't remove backup %v from %v: %v", backup.Name(), backupDir, err)
+		}
+		// We successfully removed one backup. Can we afford to prune any more?
+		numBackups--
+		if numBackups == *minRetentionCount {
+			log.Infof("Successfully pruned backup count to min_retention_count of %v.", *minRetentionCount)
+			break
+		}
+	}
+	return nil
+}
+
+func parseBackupTime(name string) (time.Time, error) {
+	// Backup names are formatted as "date.time.tablet-alias".
+	parts := strings.Split(name, ".")
+	if len(parts) != 3 {
+		return time.Time{}, fmt.Errorf("backup name not in expected format (date.time.tablet-alias): %v", name)
+	}
+	backupTime, err := time.Parse(backupTimestampFormat, fmt.Sprintf("%s.%s", parts[0], parts[1]))
+	if err != nil {
+		return time.Time{}, fmt.Errorf("can't parse timestamp from backup %q: %v", name, err)
+	}
+	return backupTime, nil
+}
+
+func shouldBackup(ctx context.Context, topoServer *topo.Server, backupStorage backupstorage.BackupStorage, backupDir string) (bool, error) {
+	backups, err := backupStorage.ListBackups(ctx, backupDir)
+	if err != nil {
+		return false, fmt.Errorf("can't list backups: %v", err)
+	}
+
+	// Check preconditions for initial_backup mode.
+	if *initialBackup {
+		// Check that no existing backups exist in this backup storage location.
+		if len(backups) > 0 {
+			return false, fmt.Errorf("refusing to upload initial backup of empty database: the shard %v/%v already has at least one backup", *initKeyspace, *initShard)
+		}
+		// Check that the shard doesn't exist.
+		_, err := topoServer.GetShard(ctx, *initKeyspace, *initShard)
+		if !topo.IsErrType(err, topo.NoNode) {
+			return false, fmt.Errorf("refusing to upload initial backup of empty database: the shard %v/%v already exists in topology", *initKeyspace, *initShard)
+		}
+		return true, nil
+	}
+
+	// We need at least one backup so we can restore first.
+	if len(backups) == 0 {
+		return false, fmt.Errorf("no existing backups to restore from; backup is not possible since -initial_backup flag was not enabled")
+	}
+	// Has it been long enough since the last backup to need a new one?
+	if *minBackupInterval == 0 {
+		// No minimum interval is set, so always backup.
+		return true, nil
+	}
+	lastBackup := backups[len(backups)-1]
+	lastBackupTime, err := parseBackupTime(lastBackup.Name())
+	if err != nil {
+		return false, fmt.Errorf("can't check last backup time: %v", err)
+	}
+	if elapsedTime := time.Since(lastBackupTime); elapsedTime < *minBackupInterval {
+		// It hasn't been long enough yet.
+		log.Infof("Skipping backup since only %v has elapsed since the last backup at %v, which is less than the min_backup_interval of %v.", elapsedTime, lastBackupTime, *minBackupInterval)
+		return false, nil
+	}
+	// It has been long enough.
+	log.Infof("The last backup was taken at %v, which is older than the min_backup_interval of %v.", lastBackupTime, *minBackupInterval)
+	return true, nil
 }

From 7e9e3ba3eb5194439630f5c8e95a8924bef03ff4 Mon Sep 17 00:00:00 2001
From: Anthony Yeh <enisoc@planetscale.com>
Date: Fri, 7 Jun 2019 12:41:27 -0700
Subject: [PATCH 7/8] vtbackup: Make -initial_backup mode idempotent.

Signed-off-by: Anthony Yeh <enisoc@planetscale.com>
---
 go/cmd/vtbackup/vtbackup.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/go/cmd/vtbackup/vtbackup.go b/go/cmd/vtbackup/vtbackup.go
index b3e5d05f77c..1a28f26b8e6 100644
--- a/go/cmd/vtbackup/vtbackup.go
+++ b/go/cmd/vtbackup/vtbackup.go
@@ -97,7 +97,7 @@ var (
 	minRetentionTime  = flag.Duration("min_retention_time", 0, "Keep each old backup for at least this long before removing it. Set to 0 to disable pruning of old backups.")
 	minRetentionCount = flag.Int("min_retention_count", 1, "Always keep at least this many of the most recent backups in this backup storage location, even if some are older than the min_retention_time. This must be at least 1 since a backup must always exist to allow new backups to be made")
 
-	initialBackup = flag.Bool("initial_backup", false, "Instead of restoring from backup, initialize an empty database with the provided init_db_sql_file and upload a backup of that for the shard. This can be used to seed a brand new shard with an initial, empty backup. This can only be done before the shard exists in topology (i.e. before any tablets are deployed), and before any backups exist for the shard")
+	initialBackup = flag.Bool("initial_backup", false, "Instead of restoring from backup, initialize an empty database with the provided init_db_sql_file and upload a backup of that for the shard, if the shard has no backups yet. This can be used to seed a brand new shard with an initial, empty backup. If any backups already exist for the shard, this will be considered a successful no-op. This can only be done before the shard exists in topology (i.e. before any tablets are deployed).")
 
 	// vttablet-like flags
 	initDbNameOverride = flag.String("init_db_name_override", "", "(init parameter) override the name of the db used by vttablet")
@@ -473,15 +473,16 @@ func shouldBackup(ctx context.Context, topoServer *topo.Server, backupStorage ba
 
 	// Check preconditions for initial_backup mode.
 	if *initialBackup {
-		// Check that no existing backups exist in this backup storage location.
-		if len(backups) > 0 {
-			return false, fmt.Errorf("refusing to upload initial backup of empty database: the shard %v/%v already has at least one backup", *initKeyspace, *initShard)
-		}
 		// Check that the shard doesn't exist.
 		_, err := topoServer.GetShard(ctx, *initKeyspace, *initShard)
 		if !topo.IsErrType(err, topo.NoNode) {
 			return false, fmt.Errorf("refusing to upload initial backup of empty database: the shard %v/%v already exists in topology", *initKeyspace, *initShard)
 		}
+		// Check if any backups for the shard exist in this backup storage location.
+		if len(backups) > 0 {
+			log.Infof("At least one backup already exists, so there's no need to seed an empty backup. Doing nothing.")
+			return false, nil
+		}
 		return true, nil
 	}
 

From 62fd75cfafb0f3f89aec057234ccdd192082660c Mon Sep 17 00:00:00 2001
From: Anthony Yeh <enisoc@planetscale.com>
Date: Fri, 7 Jun 2019 13:02:49 -0700
Subject: [PATCH 8/8] vtbackup: Add docker/k8s/vtbackup image.

Signed-off-by: Anthony Yeh <enisoc@planetscale.com>
---
 docker/k8s/Dockerfile          |  4 ++++
 docker/k8s/vtbackup/Dockerfile | 25 +++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 docker/k8s/vtbackup/Dockerfile

diff --git a/docker/k8s/Dockerfile b/docker/k8s/Dockerfile
index 744bd7700b0..704f2a865be 100644
--- a/docker/k8s/Dockerfile
+++ b/docker/k8s/Dockerfile
@@ -33,6 +33,7 @@ COPY --from=base /vt/bin/vtctlclient /vt/bin/
 COPY --from=base /vt/bin/vtgate /vt/bin/
 COPY --from=base /vt/bin/vttablet /vt/bin/
 COPY --from=base /vt/bin/vtworker /vt/bin/
+COPY --from=base /vt/bin/vtbackup /vt/bin/
 
 # copy web admin files
 COPY --from=base $VTTOP/web /vt/web/
@@ -57,6 +58,9 @@ COPY --from=base $VTTOP/config/mycnf/backup.cnf /vt/config/mycnf/
 # settings to support rbr
 COPY --from=base $VTTOP/config/mycnf/rbr.cnf /vt/config/mycnf/
 
+# recommended production settings
+COPY --from=base $VTTOP/config/mycnf/production.cnf /vt/config/mycnf/
+
 # add vitess user and add permissions
 RUN groupadd -r --gid 2000 vitess && useradd -r -g vitess --uid 1000 vitess && \
    chown -R vitess:vitess /vt;
diff --git a/docker/k8s/vtbackup/Dockerfile b/docker/k8s/vtbackup/Dockerfile
new file mode 100644
index 00000000000..05d2e9e784d
--- /dev/null
+++ b/docker/k8s/vtbackup/Dockerfile
@@ -0,0 +1,25 @@
+FROM vitess/k8s AS k8s
+
+FROM debian:stretch-slim
+
+# Set up Vitess environment (just enough to run pre-built Go binaries)
+ENV VTROOT /vt
+ENV VTDATAROOT /vtdataroot
+
+# Prepare directory structure.
+RUN mkdir -p /vt/bin && mkdir -p /vtdataroot
+
+# Copy binaries
+COPY --from=k8s /vt/bin/vtbackup /vt/bin/
+
+# Copy certs to allow https calls
+COPY --from=k8s /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
+# Copy vitess config
+COPY --from=k8s /vt/config /vt/config
+
+# add vitess user/group and add permissions
+RUN groupadd -r --gid 2000 vitess && \
+   useradd -r -g vitess --uid 1000 vitess && \
+   chown -R vitess:vitess /vt && \
+   chown -R vitess:vitess /vtdataroot