Slack vitess 2021.01.11 release

Makefile

@@ -75,6 +75,9 @@ install-testing: build
 	mkdir -p "$${PREFIX}/web/vtctld2"
 	cp -R web/vtctld2/app "$${PREFIX}/web/vtctld2"
 
+grpcvtctldclient: go/vt/proto/vtctlservice/vtctlservice.pb.go
+	make -C go/vt/vtctl/grpcvtctldclient
+
 parser:
 	make -C go/vt/sqlparser
 

docker/base/Dockerfile

@@ -25,6 +25,9 @@ FROM vitess/bootstrap:mysql57
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mariadb

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mariadb
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mariadb103

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mariadb103
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mysql56

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mysql56
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mysql80

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mysql80
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.percona

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:percona
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.percona57

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:percona57
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.percona80

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:percona80
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/lite/Dockerfile.alpine

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mariadb103 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mariadb

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mariadb AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mariadb103

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mariadb103 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mysql56

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mysql56 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mysql57

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mysql57 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mysql80

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mysql80 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.percona

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:percona AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.percona57

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:percona57 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.percona80

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:percona80 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

examples/compose/vtcompose/vtcompose.go

@@ -434,6 +434,7 @@ func applyKeyspaceDependentPatches(
 	if keyspaceData.shards < 2 {
 		tabAlias = tabAlias + 100
 		dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap, opts)
+		dockerComposeFile = applyShardPatches(dockerComposeFile, tabAlias, shard, keyspaceData, opts)
 	} else {
 		// Determine shard range
 		for i := 0; i < keyspaceData.shards; i++ {
@@ -446,6 +447,7 @@ func applyKeyspaceDependentPatches(
 			}
 			tabAlias = tabAlias + 100
 			dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap, opts)
+			dockerComposeFile = applyShardPatches(dockerComposeFile, tabAlias, shard, keyspaceData, opts)
 		}
 	}
 
@@ -475,6 +477,39 @@ func applyDockerComposePatches(
 	return dockerComposeFile
 }
 
+func applyShardPatches(
+	dockerComposeFile []byte,
+	tabAlias int,
+	shard string,
+	keyspaceData keyspaceInfo,
+	opts vtOptions,
+) []byte {
+	dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultShard(tabAlias, shard, keyspaceData, opts))
+	return dockerComposeFile
+}
+
+func generateDefaultShard(tabAlias int, shard string, keyspaceData keyspaceInfo, opts vtOptions) string {
+	aliases := []int{tabAlias + 1} // master alias, e.g. 201
+	for i := 0; i < keyspaceData.replicaTablets; i++ {
+		aliases = append(aliases, tabAlias+2+i) // replica aliases, e.g. 202, 203, ...
+	}
+	tabletDepends := make([]string, len(aliases))
+	for i, tabletId := range aliases {
+		tabletDepends[i] = fmt.Sprintf("vttablet%d: {condition : service_healthy}", tabletId)
+	}
+	// Wait on all shard tablets to be healthy
+	dependsOn := "depends_on: {" + strings.Join(tabletDepends, ", ") + "}"
+
+	return fmt.Sprintf(`
+- op: add
+  path: /services/init_shard_master%[2]d
+  value:
+    image: vitess/base
+    command: ["sh", "-c", "$$VTROOT/bin/vtctl %[5]s InitShardMaster -force %[4]s/%[3]s %[6]s-%[2]d "]
+    %[1]s
+`, dependsOn, aliases[0], shard, keyspaceData.keyspace, opts.topologyFlags, opts.cell)
+}
+
 func applyTabletPatches(
 	dockerComposeFile []byte,
 	tabAlias int,
@@ -674,7 +709,7 @@ func generatePrimaryVIndex(tableName, column, name string) string {
 	return fmt.Sprintf(`
 [{"op": "add",
 "path": "/tables/%[1]s",
-"value": 
+"value":
   {"column_vindexes": [
     {
       "column": "%[2]s",

examples/compose/vttablet-up.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 
 # Copyright 2019 The Vitess Authors.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -120,8 +120,6 @@ $VTROOT/bin/vtctl $TOPOLOGY_FLAGS AddCellInfo -root vitess/$CELL -server_address
 $VTROOT/bin/vtctl $TOPOLOGY_FLAGS CreateKeyspace $keyspace || true
 $VTROOT/bin/vtctl $TOPOLOGY_FLAGS CreateShard $keyspace/$shard || true
 
-$VTROOT/bin/vtctl $TOPOLOGY_FLAGS InitTablet -shard $shard -keyspace $keyspace -grpc_port $grpc_port -port $web_port -allow_master_override $alias $tablet_role
-
 #Populate external db conditional args
 if [ "$external" = "1" ]; then
     if [ $role = "master" ]; then

examples/local/scripts/vtctld-up.sh

@@ -28,7 +28,7 @@ vtctld \
  -cell $cell \
  -workflow_manager_init \
  -workflow_manager_use_election \
- -service_map 'grpc-vtctl' \
+ -service_map 'grpc-vtctl,grpc-vtctld' \
  -backup_storage_implementation file \
  -file_backup_storage_root $VTDATAROOT/backups \
  -log_dir $VTDATAROOT/tmp \

go.mod

@@ -17,7 +17,6 @@ require (
 	github.com/coreos/bbolt v1.3.2 // indirect
 	github.com/coreos/etcd v3.3.10+incompatible
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
-	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
 	github.com/corpix/uarand v0.1.1 // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible
 	github.com/go-sql-driver/mysql v1.5.0
@@ -28,8 +27,9 @@ require (
 	github.com/golang/protobuf v1.3.2
 	github.com/golang/snappy v0.0.1
 	github.com/google/go-cmp v0.4.0
+	github.com/google/uuid v1.1.1
 	github.com/googleapis/gnostic v0.2.0 // indirect
-	github.com/gorilla/websocket v1.4.0
+	github.com/gorilla/websocket v1.4.2
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/hashicorp/consul/api v1.5.0
@@ -66,15 +66,14 @@ require (
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/cobra v0.0.5
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
 	github.com/tchap/go-patricia v0.0.0-20160729071656-dd168db6051b
 	github.com/tebeka/selenium v0.9.9
 	github.com/tinylib/msgp v1.1.1 // indirect
-	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-client-go v2.16.0+incompatible
 	github.com/uber/jaeger-lib v2.0.0+incompatible // indirect
-	github.com/ugorji/go v1.1.7 // indirect
 	github.com/z-division/go-zookeeper v0.0.0-20190128072838-6d7457066b9b
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
 	golang.org/x/lint v0.0.0-20190409202823-959b441ac422