[cherry pick] Apply GRPC error processing change from upstream

docker/base/Dockerfile

@@ -25,6 +25,9 @@ FROM vitess/bootstrap:mysql57
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mariadb

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mariadb
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mariadb103

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mariadb103
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mysql56

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mysql56
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mysql80

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mysql80
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.percona

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:percona
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.percona57

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:percona57
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.percona80

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:percona80
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/lite/Dockerfile.alpine

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mariadb103 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mariadb

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mariadb AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mariadb103

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mariadb103 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mysql56

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mysql56 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mysql57

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mysql57 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mysql80

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mysql80 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.percona

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:percona AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.percona57

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:percona57 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.percona80

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:percona80 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

examples/compose/vtcompose/vtcompose.go

@@ -434,6 +434,7 @@ func applyKeyspaceDependentPatches(
  if keyspaceData.shards < 2 {
  tabAlias = tabAlias + 100
  dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap, opts)
+ dockerComposeFile = applyShardPatches(dockerComposeFile, tabAlias, shard, keyspaceData, opts)
  } else {
  // Determine shard range
  for i := 0; i < keyspaceData.shards; i++ {
@@ -446,6 +447,7 @@ func applyKeyspaceDependentPatches(
  }
  tabAlias = tabAlias + 100
  dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap, opts)
+ dockerComposeFile = applyShardPatches(dockerComposeFile, tabAlias, shard, keyspaceData, opts)
  }
  }
 
@@ -475,6 +477,39 @@ func applyDockerComposePatches(
  return dockerComposeFile
 }
 
+func applyShardPatches(
+ dockerComposeFile []byte,
+ tabAlias int,
+ shard string,
+ keyspaceData keyspaceInfo,
+ opts vtOptions,
+) []byte {
+ dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultShard(tabAlias, shard, keyspaceData, opts))
+ return dockerComposeFile
+}
+
+func generateDefaultShard(tabAlias int, shard string, keyspaceData keyspaceInfo, opts vtOptions) string {
+ aliases := []int{tabAlias + 1} // master alias, e.g. 201
+ for i := 0; i < keyspaceData.replicaTablets; i++ {
+ aliases = append(aliases, tabAlias+2+i) // replica aliases, e.g. 202, 203, ...
+ }
+ tabletDepends := make([]string, len(aliases))
+ for i, tabletId := range aliases {
+ tabletDepends[i] = fmt.Sprintf("vttablet%d: {condition : service_healthy}", tabletId)
+ }
+ // Wait on all shard tablets to be healthy
+ dependsOn := "depends_on: {" + strings.Join(tabletDepends, ", ") + "}"
+
+ return fmt.Sprintf(`
+- op: add
+ path: /services/init_shard_master%[2]d
+ value:
+ image: vitess/base
+ command: ["sh", "-c", "$$VTROOT/bin/vtctl %[5]s InitShardMaster -force %[4]s/%[3]s %[6]s-%[2]d "]
+ %[1]s
+`, dependsOn, aliases[0], shard, keyspaceData.keyspace, opts.topologyFlags, opts.cell)
+}
+
 func applyTabletPatches(
  dockerComposeFile []byte,
  tabAlias int,
@@ -674,7 +709,7 @@ func generatePrimaryVIndex(tableName, column, name string) string {
  return fmt.Sprintf(`
 [{"op": "add",
 "path": "/tables/%[1]s",
-"value": 
+"value":
  {"column_vindexes": [
  {
  "column": "%[2]s",

examples/compose/vttablet-up.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 
 # Copyright 2019 The Vitess Authors.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -120,8 +120,6 @@ $VTROOT/bin/vtctl $TOPOLOGY_FLAGS AddCellInfo -root vitess/$CELL -server_address
 $VTROOT/bin/vtctl $TOPOLOGY_FLAGS CreateKeyspace $keyspace || true
 $VTROOT/bin/vtctl $TOPOLOGY_FLAGS CreateShard $keyspace/$shard || true
 
-$VTROOT/bin/vtctl $TOPOLOGY_FLAGS InitTablet -shard $shard -keyspace $keyspace -grpc_port $grpc_port -port $web_port -allow_master_override $alias $tablet_role
-
 #Populate external db conditional args
 if [ "$external" = "1" ]; then
  if [ $role = "master" ]; then

go/cmd/vtbackup/vtbackup.go

@@ -306,8 +306,6 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
  return fmt.Errorf("no backup found; not starting up empty since -initial_backup flag was not enabled")
  }
  restorePos = mysql.Position{}
- case mysqlctl.ErrExistingDB:
- return fmt.Errorf("can't run vtbackup because data directory is not empty")
  default:
  return fmt.Errorf("can't restore from backup: %v", err)
  }

go/mysql/sql_error.go

@@ -120,7 +120,7 @@ func NewSQLErrorFromError(err error) error {
  case vtrpcpb.Code_UNAUTHENTICATED:
  num = ERAccessDeniedError
  case vtrpcpb.Code_RESOURCE_EXHAUSTED:
- num = ERTooManyUserConnections
+ num = demuxResourceExhaustedErrors(err.Error())
  case vtrpcpb.Code_FAILED_PRECONDITION:
  num = ERUnknownError
  case vtrpcpb.Code_ABORTED:
@@ -161,3 +161,14 @@ func NewSQLErrorFromError(err error) error {
  }
  return serr
 }
+
+var isGRPCOverflowRE = regexp.MustCompile(`.*grpc: received message larger than max \(\d+ vs. \d+\)`)
+
+func demuxResourceExhaustedErrors(msg string) int {
+ switch {
+ case isGRPCOverflowRE.Match([]byte(msg)):
+ return ERNetPacketTooLarge
+ default:
+ return ERTooManyUserConnections
+ }
+}

go/mysql/sql_error_test.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2020 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mysql
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestDumuxResourceExhaustedErrors(t *testing.T) {
+ type testCase struct {
+ msg string
+ want int
+ }
+
+ cases := []testCase{
+ testCase{"misc", ERTooManyUserConnections},
+ testCase{"grpc: received message larger than max (99282 vs. 1234): trailer", ERNetPacketTooLarge},
+ testCase{"grpc: received message larger than max (1234 vs. 1234)", ERNetPacketTooLarge},
+ testCase{"header: grpc: received message larger than max (1234 vs. 1234)", ERNetPacketTooLarge},
+ // This should be explicitly handled by returning ERNetPacketTooLarge from the execturo directly
+ // and therefore shouldn't need to be teased out of another error.
+ testCase{"in-memory row count exceeded allowed limit of 13", ERTooManyUserConnections},
+ }
+
+ for _, c := range cases {
+ got := demuxResourceExhaustedErrors(c.msg)
+ assert.Equalf(t, c.want, got, c.msg)
+ }
+}

go/pools/numbered.go

@@ -131,13 +131,15 @@ func (nu *Numbered) Get(id int64, purpose string) (val interface{}, err error) {
 }
 
 // Put unlocks a resource for someone else to use.
-func (nu *Numbered) Put(id int64) {
+func (nu *Numbered) Put(id int64, updateTime bool) {
  nu.mu.Lock()
  defer nu.mu.Unlock()
  if nw, ok := nu.resources[id]; ok {
  nw.inUse = false
  nw.purpose = ""
- nw.timeUsed = time.Now()
+ if updateTime {
+ nw.timeUsed = time.Now()
+ }
  }
 }
 
@@ -162,7 +164,7 @@ func (nu *Numbered) GetOutdated(age time.Duration, purpose string) (vals []inter
  if nw.inUse || !nw.enforceTimeout {
  continue
  }
- if nw.timeCreated.Add(age).Sub(now) <= 0 {
+ if nw.timeUsed.Add(age).Sub(now) <= 0 {
  nw.inUse = true
  nw.purpose = purpose
  vals = append(vals, nw.val)