merge master

Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>

doc/V3HighLevelDesign.md

@@ -1135,7 +1135,7 @@ The scoping rule works in our favor here because this type of name generation co
 
 For the sake of simplicity, we’ll not allow dependencies to use non-standard column names. For example, we’ll fail a query like this:
 
-`select * from (select a, count(*) from t1) t where t.`count(*)` > 0`
+`select * from (select a, count(*) from t1) t where t.count(*) > 0`
 
 This should instead be rewritten as:
 

docker/lite/Dockerfile.ubi7.mysql57

@@ -51,7 +51,7 @@ RUN curl -L --retry-delay 10 --retry 3 -o /tmp/libev.rpm http://mirror.centos.or
  && curl -L --retry-delay 10 --retry 3 -o /tmp/gperf.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/gperftools-libs-2.6.1-1.el7.x86_64.rpm \
  && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/numactl-libs-2.0.12-5.el7.x86_64.rpm \
  && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/sysstat-10.1.5-19.el7.x86_64.rpm \
- && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-4.el7.x86_64.rpm \
+ && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-6.el7.x86_64.rpm \
  && rpm -i --nodeps /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm \
  && rm -f /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm
 RUN yum update \

docker/lite/Dockerfile.ubi7.mysql80

@@ -51,7 +51,7 @@ RUN curl -L --retry-delay 10 --retry 3 -o /tmp/libev.rpm http://mirror.centos.or
  && curl -L --retry-delay 10 --retry 3 -o /tmp/gperf.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/gperftools-libs-2.6.1-1.el7.x86_64.rpm \
  && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/numactl-libs-2.0.12-5.el7.x86_64.rpm \
  && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/sysstat-10.1.5-19.el7.x86_64.rpm \
- && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-4.el7.x86_64.rpm \
+ && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-6.el7.x86_64.rpm \
  && rpm -i --nodeps /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm \
  && rm -f /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm
 RUN yum update \

docker/lite/Dockerfile.ubi7.percona57

@@ -49,7 +49,7 @@ RUN curl -L --retry-delay 10 --retry 3 -o /tmp/libev.rpm http://mirror.centos.or
  && curl -L --retry-delay 10 --retry 3 -o /tmp/gperf.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/gperftools-libs-2.6.1-1.el7.x86_64.rpm \
  && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/numactl-libs-2.0.12-5.el7.x86_64.rpm \
  && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/sysstat-10.1.5-19.el7.x86_64.rpm \
- && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-4.el7.x86_64.rpm \
+ && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-6.el7.x86_64.rpm \
  && rpm -i --nodeps /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm \
  && rm -f /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm
 RUN yum update \

docker/lite/Dockerfile.ubi7.percona80

@@ -49,7 +49,7 @@ RUN curl -L --retry-delay 10 --retry 3 -o /tmp/libev.rpm http://mirror.centos.or
  && curl -L --retry-delay 10 --retry 3 -o /tmp/gperf.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/gperftools-libs-2.6.1-1.el7.x86_64.rpm \
  && curl -L --retry-delay 10 --retry 3 -o /tmp/numactl.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/numactl-libs-2.0.12-5.el7.x86_64.rpm \
  && curl -L --retry-delay 10 --retry 3 -o /tmp/sysstat.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/sysstat-10.1.5-19.el7.x86_64.rpm \
- && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-4.el7.x86_64.rpm \
+ && curl -L --retry-delay 10 --retry 3 -o /tmp/strace.rpm http://mirror.centos.org/centos/7/os/x86_64/Packages/strace-4.24-6.el7.x86_64.rpm \
  && rpm -i --nodeps /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm \
  && rm -f /tmp/libev.rpm /tmp/gperf.rpm /tmp/numactl.rpm /tmp/sysstat.rpm /tmp/strace.rpm
 RUN yum update \

examples/operator/101_initial_cluster.yaml

@@ -8,10 +8,10 @@ metadata:
   name: example
 spec:
   images:
-    vtctld: vitess/lite:v6.0.20-20200429
-    vtgate: vitess/lite:v6.0.20-20200429
-    vttablet: vitess/lite:v6.0.20-20200429
-    vtbackup: vitess/lite:v6.0.20-20200429
+    vtctld: vitess/lite:v8.0.0
+    vtgate: vitess/lite:v8.0.0
+    vttablet: vitess/lite:v8.0.0
+    vtbackup: vitess/lite:v8.0.0
     mysqld:
       mysql56Compatible: vitess/lite:v6.0.20-20200429
     mysqldExporter: prom/mysqld-exporter:v0.11.0

examples/operator/201_customer_tablets.yaml

@@ -4,10 +4,10 @@ metadata:
   name: example
 spec:
   images:
-    vtctld: vitess/lite:v6.0.20-20200429
-    vtgate: vitess/lite:v6.0.20-20200429
-    vttablet: vitess/lite:v6.0.20-20200429
-    vtbackup: vitess/lite:v6.0.20-20200429
+    vtctld: vitess/lite:v8.0.0
+    vtgate: vitess/lite:v8.0.0
+    vttablet: vitess/lite:v8.0.0
+    vtbackup: vitess/lite:v8.0.0
     mysqld:
       mysql56Compatible: vitess/lite:v6.0.20-20200429
     mysqldExporter: prom/mysqld-exporter:v0.11.0

examples/operator/302_new_shards.yaml

@@ -4,12 +4,12 @@ metadata:
   name: example
 spec:
   images:
-    vtctld: us.gcr.io/planetscale-vitess/lite:2020-04-24.228e6fe
-    vtgate: us.gcr.io/planetscale-vitess/lite:2020-04-24.228e6fe
-    vttablet: us.gcr.io/planetscale-vitess/lite:2020-04-24.228e6fe
-    vtbackup: us.gcr.io/planetscale-vitess/lite:2020-04-24.228e6fe
+    vtctld: vitess/lite:v8.0.0
+    vtgate: vitess/lite:v8.0.0
+    vttablet: vitess/lite:v8.0.0
+    vtbackup: vitess/lite:v8.0.0
     mysqld:
-      mysql56Compatible: us.gcr.io/planetscale-vitess/lite:2020-04-24.228e6fe
+      mysql56Compatible: vitess/lite:v8.0.0
     mysqldExporter: prom/mysqld-exporter:v0.11.0
   cells:
   - name: zone1

examples/operator/306_down_shard_0.yaml

@@ -4,10 +4,10 @@ metadata:
   name: example
 spec:
   images:
-    vtctld: us.gcr.io/planetscale-vitess/lite:2020-04-24.228e6fe
-    vtgate: us.gcr.io/planetscale-vitess/lite:2020-04-24.228e6fe
-    vttablet: us.gcr.io/planetscale-vitess/lite:2020-04-24.228e6fe
-    vtbackup: us.gcr.io/planetscale-vitess/lite:2020-04-24.228e6fe
+    vtctld: vitess/lite:v8.0.0
+    vtgate: vitess/lite:v8.0.0
+    vttablet: vitess/lite:v8.0.0
+    vtbackup: vitess/lite:v8.0.0
     mysqld:
       mysql56Compatible: us.gcr.io/planetscale-vitess/lite:2020-04-24.228e6fe
     mysqldExporter: prom/mysqld-exporter:v0.11.0

examples/operator/operator.yaml

@@ -5773,7 +5773,7 @@ spec:
               fieldPath: metadata.name
         - name: OPERATOR_NAME
           value: vitess-operator
-        image: planetscale/vitess-operator:v2.0.0
+        image: planetscale/vitess-operator:v2.2.0
         imagePullPolicy: IfNotPresent
         name: vitess-operator
         resources:

go.mod

@@ -58,6 +58,7 @@ require (
 	github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11
 	github.com/mattn/go-sqlite3 v1.14.0
 	github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1
+	github.com/mitchellh/go-ps v1.0.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.0 // indirect
 	github.com/mitchellh/mapstructure v1.2.3 // indirect
 	github.com/montanaflynn/stats v0.6.3

go.sum

@@ -476,6 +476,8 @@ github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXx
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-testing-interface v1.14.0 h1:/x0XQ6h+3U3nAyk1yx+bHPURrKa9sVVvYbuqZ7pIAtI=
 github.com/mitchellh/go-testing-interface v1.14.0/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=

go/test/endtoend/onlineddl/onlineddl_test.go

@@ -114,6 +114,7 @@ func TestMain(m *testing.M) {
 
 		clusterInstance.VtTabletExtraArgs = []string{
 			"-migration_check_interval", "5s",
+			"-gh-ost-path", os.Getenv("VITESS_ENDTOEND_GH_OST_PATH"), // leave env variable empty/unset to get the default behavior. Override in Mac.
 		}
 		clusterInstance.VtGateExtraArgs = []string{
 			"-ddl_strategy", "gh-ost",

go/test/endtoend/vreplication/vreplication_test.go

@@ -232,13 +232,10 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl
 
 	insertQuery2 = "insert into customer(name, cid) values('tempCustomer4', 102)" //ID 102, hence due to reverse_bits in shard -80
 	require.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab1, "customer", insertQuery2, matchInsertQuery2))
-	time.Sleep(1 * time.Second) // wait for vreplication to catchup
-
 	reverseKsWorkflow := "product.p2c_reverse"
 	if testReverse {
 		//Reverse Replicate
 		switchReads(t, allCellNames, reverseKsWorkflow)
-		printShardPositions(vc, ksShards)
 		switchWrites(t, reverseKsWorkflow)
 
 		insertQuery1 = "insert into customer(cid, name) values(1002, 'tempCustomer5')"
@@ -249,11 +246,10 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl
 		insertQuery1 = "insert into customer(cid, name) values(1004, 'tempCustomer7')"
 		require.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab2, "customer", insertQuery1, matchInsertQuery1))
 
-		time.Sleep(1 * time.Second) // wait for vreplication to catchup
-
 		//Go forward again
 		switchReads(t, allCellNames, ksWorkflow)
 		switchWrites(t, ksWorkflow)
+
 		dropSourcesDryRun(t, ksWorkflow, false, dryRunResultsDropSourcesDropCustomerShard)
 		dropSourcesDryRun(t, ksWorkflow, true, dryRunResultsDropSourcesRenameCustomerShard)
 
@@ -653,15 +649,11 @@ func switchWritesDryRun(t *testing.T, ksWorkflow string, dryRunResults []string)
 	validateDryRunResults(t, output, dryRunResults)
 }
 
-func switchWrites(t *testing.T, ksWorkflow string) {
-	const SwitchWritesTimeout = "91s" // max: 3 tablet picker 30s waits + 1
-	output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchWrites",
-		"-filtered_replication_wait_time="+SwitchWritesTimeout, ksWorkflow)
-
+func printSwitchWritesExtraDebug(t *testing.T, ksWorkflow, msg string) {
 	// Temporary code: print lots of info for debugging occasional flaky failures in customer reshard in CI for multicell test
 	debug := true
-	if debug && strings.Contains(ksWorkflow, ".p2c") {
-		fmt.Printf("------------------- START Extra debug info for a p2c SwitchWrites\n")
+	if debug {
+		fmt.Printf("------------------- START Extra debug info %s SwitchWrites %s\n", msg, ksWorkflow)
 		ksShards := []string{"product/0", "customer/-80", "customer/80-"}
 		printShardPositions(vc, ksShards)
 		custKs := vc.Cells[defaultCell.Name].Keyspaces["customer"]
@@ -683,8 +675,19 @@ func switchWrites(t *testing.T, ksWorkflow string) {
 					tab.Cell, tab.Keyspace, tab.Shard, tab.TabletUID, query, qr.Rows)
 			}
 		}
-		fmt.Printf("------------------- END Extra debug info for a p2c SwitchWrites\n")
+		fmt.Printf("------------------- END Extra debug info %s SwitchWrites %s\n", msg, ksWorkflow)
+	}
+}
+
+func switchWrites(t *testing.T, ksWorkflow string) {
+	const SwitchWritesTimeout = "91s" // max: 3 tablet picker 30s waits + 1
+	output, err := vc.VtctlClient.ExecuteCommandWithOutput("SwitchWrites",
+		"-filtered_replication_wait_time="+SwitchWritesTimeout, ksWorkflow)
+	if output != "" {
+		fmt.Printf("Output of SwitchWrites for %s:\n++++++\n%s\n--------\n", ksWorkflow, output)
 	}
+	//printSwitchWritesExtraDebug is useful when debugging failures in SwitchWrites due to corner cases/races
+	_ = printSwitchWritesExtraDebug
 	if err != nil {
 		require.FailNow(t, fmt.Sprintf("SwitchWrites Error: %s: %s", err, output))
 	}

go/test/endtoend/vtgate/misc_test.go

@@ -479,6 +479,23 @@ func TestCreateIndex(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestCreateView(t *testing.T) {
+	// The test wont work since we cant change the vschema without reloading the vtgate.
+	t.Skip()
+	defer cluster.PanicHandler(t)
+	ctx := context.Background()
+	conn, err := mysql.Connect(ctx, &vtParams)
+	require.NoError(t, err)
+	defer conn.Close()
+	defer exec(t, conn, `delete from t1`)
+	// Test that create view works and the output is as expected
+	exec(t, conn, `create view v1 as select * from t1`)
+	exec(t, conn, `insert into t1(id1, id2) values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)`)
+	// This wont work, since ALTER VSCHEMA ADD TABLE is only supported for unsharded keyspaces
+	exec(t, conn, "alter vschema add table v1")
+	assertMatches(t, conn, "select * from v1", `[[INT64(1) INT64(1)] [INT64(2) INT64(2)] [INT64(3) INT64(3)] [INT64(4) INT64(4)] [INT64(5) INT64(5)]]`)
+}
+
 func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
 	t.Helper()
 	qr := exec(t, conn, query)

go/test/endtoend/vtgate/unsharded/main_test.go

@@ -193,6 +193,24 @@ func TestInsertAllDefaults(t *testing.T) {
 	assertMatches(t, conn, `select * from allDefaults`, "[[INT64(1) NULL]]")
 }
 
+func TestCreateViewUnsharded(t *testing.T) {
+	defer cluster.PanicHandler(t)
+	ctx := context.Background()
+	vtParams := mysql.ConnParams{
+		Host: "localhost",
+		Port: clusterInstance.VtgateMySQLPort,
+	}
+	conn, err := mysql.Connect(ctx, &vtParams)
+	require.NoError(t, err)
+	defer conn.Close()
+	defer exec(t, conn, `delete from t1`)
+	// Test that create view works and the output is as expected
+	_, err = conn.ExecuteFetch(`create view v1 as select * from t1`, 1000, true)
+	require.NoError(t, err)
+	exec(t, conn, `insert into t1(c1, c2, c3, c4) values (300,100,300,'abc'),(30,10,30,'ac'),(3,0,3,'a')`)
+	assertMatches(t, conn, "select * from v1", `[[INT64(3) INT64(0) INT64(3) VARCHAR("a")] [INT64(30) INT64(10) INT64(30) VARCHAR("ac")] [INT64(300) INT64(100) INT64(300) VARCHAR("abc")]]`)
+}
+
 func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
 	t.Helper()
 	qr, err := conn.ExecuteFetch(query, 1000, true)

go/vt/discovery/healthcheck.go

@@ -397,6 +397,11 @@ func (hc *HealthCheckImpl) deleteTablet(tablet *topodata.Tablet) {
 		return
 	}
 	delete(ths, tabletAlias)
+	// delete from healthy list
+	healthy, ok := hc.healthy[key]
+	if ok && len(healthy) > 0 {
+		hc.recomputeHealthy(key)
+	}
 }
 
 func (hc *HealthCheckImpl) updateHealth(th *TabletHealth, shr *query.StreamHealthResponse, currentTarget *query.Target, trivialNonMasterUpdate bool, isMasterUpdate bool, isMasterChange bool) {
@@ -446,27 +451,11 @@ func (hc *HealthCheckImpl) updateHealth(th *TabletHealth, shr *query.StreamHealt
 		// Tablets from other cells for non-master targets should not trigger a re-sort;
 		// they should also be excluded from healthy list.
 		if shr.Target.TabletType != topodata.TabletType_MASTER && hc.isIncluded(shr.Target.TabletType, shr.TabletAlias) {
-			all := hc.healthData[targetKey]
-			allArray := make([]*TabletHealth, 0, len(all))
-			for _, s := range all {
-				// Only tablets in same cell / cellAlias are included in healthy list.
-				if hc.isIncluded(s.Tablet.Type, s.Tablet.Alias) {
-					allArray = append(allArray, s)
-				}
-			}
-			hc.healthy[targetKey] = FilterStatsByReplicationLag(allArray)
+			hc.recomputeHealthy(targetKey)
 		}
 		if targetChanged && currentTarget.TabletType != topodata.TabletType_MASTER && hc.isIncluded(shr.Target.TabletType, shr.TabletAlias) { // also recompute old target's healthy list
 			oldTargetKey := hc.keyFromTarget(currentTarget)
-			all := hc.healthData[oldTargetKey]
-			allArray := make([]*TabletHealth, 0, len(all))
-			for _, s := range all {
-				// Only tablets in same cell / cellAlias are included in healthy list.
-				if hc.isIncluded(s.Tablet.Type, s.Tablet.Alias) {
-					allArray = append(allArray, s)
-				}
-			}
-			hc.healthy[oldTargetKey] = FilterStatsByReplicationLag(allArray)
+			hc.recomputeHealthy(oldTargetKey)
 		}
 	}
 	if isMasterChange {
@@ -478,6 +467,18 @@ func (hc *HealthCheckImpl) updateHealth(th *TabletHealth, shr *query.StreamHealt
 
 }
 
+func (hc *HealthCheckImpl) recomputeHealthy(key keyspaceShardTabletType) {
+	all := hc.healthData[key]
+	allArray := make([]*TabletHealth, 0, len(all))
+	for _, s := range all {
+		// Only tablets in same cell / cellAlias are included in healthy list.
+		if hc.isIncluded(s.Tablet.Type, s.Tablet.Alias) {
+			allArray = append(allArray, s)
+		}
+	}
+	hc.healthy[key] = FilterStatsByReplicationLag(allArray)
+}
+
 // Subscribe adds a listener. Used by vtgate buffer to learn about master changes.
 func (hc *HealthCheckImpl) Subscribe() chan *TabletHealth {
 	hc.subMu.Lock()

go/vt/discovery/healthcheck_test.go

@@ -514,6 +514,48 @@ func TestWaitForAllServingTablets(t *testing.T) {
 	KeyspacesToWatch = []string{}
 }
 
+// TestRemoveTablet tests the behavior when a tablet goes away.
+func TestRemoveTablet(t *testing.T) {
+	ts := memorytopo.NewServer("cell")
+	hc := createTestHc(ts)
+	defer hc.Close()
+	tablet := createTestTablet(0, "cell", "a")
+	tablet.Type = topodatapb.TabletType_REPLICA
+	input := make(chan *querypb.StreamHealthResponse)
+	createFakeConn(tablet, input)
+
+	// create a channel and subscribe to healthcheck
+	resultChan := hc.Subscribe()
+	hc.AddTablet(tablet)
+	// there will be a first result, get and discard it
+	<-resultChan
+
+	shr := &querypb.StreamHealthResponse{
+		TabletAlias:                         tablet.Alias,
+		Target:                              &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
+		Serving:                             true,
+		TabletExternallyReparentedTimestamp: 0,
+		RealtimeStats:                       &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
+	}
+	want := []*TabletHealth{{
+		Tablet:              tablet,
+		Target:              &querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA},
+		Serving:             true,
+		Stats:               &querypb.RealtimeStats{SecondsBehindMaster: 1, CpuUsage: 0.2},
+		MasterTermStartTime: 0,
+	}}
+	input <- shr
+	<-resultChan
+	// check it's there
+	a := hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA})
+	mustMatch(t, want, a, "unexpected result")
+
+	// delete the tablet
+	hc.RemoveTablet(tablet)
+	a = hc.GetHealthyTabletStats(&querypb.Target{Keyspace: "k", Shard: "s", TabletType: topodatapb.TabletType_REPLICA})
+	assert.Empty(t, a, "wrong result, expected empty list")
+}
+
 // TestGetHealthyTablets tests the functionality of GetHealthyTabletStats.
 func TestGetHealthyTablets(t *testing.T) {
 	ts := memorytopo.NewServer("cell")