decider: endtoend test infrastructure + tests

Makefile

@@ -43,6 +43,7 @@ endif
 # Safe, since this code isn't performance critical.
 export CGO_CFLAGS := -O1
 
+# regenerate rice-box.go when any of the .cnf files change
 embed_config:
  cd go/vt/mysqlctl
  go run github.com/GeertJohan/go.rice/rice embed-go

config/mycnf/default-fast.cnf

@@ -20,3 +20,6 @@ innodb_doublewrite=0
 # https://github.com/vitessio/vitess/issues/5396
 
 sql_mode = STRICT_TRANS_TABLES
+
+# set a short heartbeat interval in order to detect failures quickly
+slave_net_timeout = 4

go/test/endtoend/cluster/cluster_process.go

@@ -72,6 +72,7 @@ type LocalProcessCluster struct {
  VtgateProcess VtgateProcess
  VtworkerProcess VtworkerProcess
  VtbackupProcess VtbackupProcess
+ OrcProcess *OrchestratorProcess
 
  nextPortForProcess int
 
@@ -349,11 +350,11 @@ func (cluster *LocalProcessCluster) StartKeyspace(keyspace Keyspace, shardNames
  return
 }
 
-// LaunchCluster creates the skeleton for a cluster by creating keyspace
+// SetupCluster creates the skeleton for a cluster by creating keyspace
 // shards and initializing tablets and mysqlctl processes.
 // This does not start any process and user have to explicitly start all
 // the required services (ex topo, vtgate, mysql and vttablet)
-func (cluster *LocalProcessCluster) LaunchCluster(keyspace *Keyspace, shards []Shard) (err error) {
+func (cluster *LocalProcessCluster) SetupCluster(keyspace *Keyspace, shards []Shard) (err error) {
 
  log.Infof("Starting keyspace: %v", keyspace.Name)
 
@@ -504,6 +505,12 @@ func (cluster *LocalProcessCluster) Teardown() {
  log.Errorf("Error in vtgate teardown: %v", err)
  }
 
+ if cluster.OrcProcess != nil {
+ if err := cluster.OrcProcess.TearDown(); err != nil {
+ log.Errorf("Error in orchestrator teardown: %v", err)
+ }
+ }
+
  var mysqlctlProcessList []*exec.Cmd
  for _, keyspace := range cluster.Keyspaces {
  for _, shard := range keyspace.Shards {
@@ -667,6 +674,17 @@ func (cluster *LocalProcessCluster) NewVttabletInstance(tabletType string, UID i
  }
 }
 
+// NewOrcProcess creates a new OrchestratorProcess object
+func (cluster *LocalProcessCluster) NewOrcProcess(configFile string) *OrchestratorProcess {
+ base := VtctlProcessInstance(cluster.TopoProcess.Port, cluster.Hostname)
+ base.Binary = "orchestrator"
+ return &OrchestratorProcess{
+ VtctlProcess: *base,
+ LogDir: cluster.TmpDirectory,
+ Config: configFile,
+ }
+}
+
 // VtprocessInstanceFromVttablet creates a new vttablet object
 func (cluster *LocalProcessCluster) VtprocessInstanceFromVttablet(tablet *Vttablet, shardName string, ksName string) *VttabletProcess {
  return VttabletProcessInstance(tablet.HTTPPort,
@@ -707,6 +725,10 @@ func (cluster *LocalProcessCluster) StartVttablet(tablet *Vttablet, servingStatu
  return tablet.VttabletProcess.Setup()
 }
 
+//func (cluster *LocalProcessCluster) NewOrcInstance() OrchestratorProcess {
+//
+//}
+
 func getCoveragePath(fileName string) string {
  covDir := os.Getenv("COV_DIR")
  if covDir == "" {

go/test/endtoend/cluster/orchestrator_process.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2020 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+*/
+
+package cluster
+
+import (
+ "os"
+ "os/exec"
+ "path"
+ "strings"
+ "syscall"
+ "time"
+
+ "vitess.io/vitess/go/vt/log"
+)
+
+// OrchestratorProcess is a test struct for running
+// orchestrator as a separate process for testing
+type OrchestratorProcess struct {
+ VtctlProcess
+ LogDir string
+ ExtraArgs []string
+ Config string
+ proc *exec.Cmd
+ exit chan error
+}
+
+// Setup starts orc process with required arguements
+func (orc *OrchestratorProcess) Setup() (err error) {
+
+ /* minimal command line arguments:
+ $ orchestrator -topo_implementation etcd2 -topo_global_server_address localhost:2379 -topo_global_root /vitess/global
+ -config config/orchestrator/default.json -alsologtostderr http
+ */
+ orc.proc = exec.Command(
+ orc.Binary,
+ "-topo_implementation", orc.TopoImplementation,
+ "-topo_global_server_address", orc.TopoGlobalAddress,
+ "-topo_global_root", orc.TopoGlobalRoot,
+ "-config", orc.Config,
+ )
+ if *isCoverage {
+ orc.proc.Args = append(orc.proc.Args, "-test.coverprofile="+getCoveragePath("orc.out"))
+ }
+
+ orc.proc.Args = append(orc.proc.Args, orc.ExtraArgs...)
+ orc.proc.Args = append(orc.proc.Args, "-alsologtostderr", "http")
+
+ errFile, _ := os.Create(path.Join(orc.LogDir, "orc-stderr.txt"))
+ orc.proc.Stderr = errFile
+
+ orc.proc.Env = append(orc.proc.Env, os.Environ()...)
+
+ log.Infof("Running orc with command: %v", strings.Join(orc.proc.Args, " "))
+
+ err = orc.proc.Start()
+ if err != nil {
+ return
+ }
+
+ orc.exit = make(chan error)
+ go func() {
+ if orc.proc != nil {
+ orc.exit <- orc.proc.Wait()
+ }
+ }()
+
+ return nil
+}
+
+// TearDown shuts down the running orchestrator service
+func (orc *OrchestratorProcess) TearDown() error {
+ if orc.proc == nil || orc.exit == nil {
+ return nil
+ }
+ // Attempt graceful shutdown with SIGTERM first
+ _ = orc.proc.Process.Signal(syscall.SIGTERM)
+
+ select {
+ case <-orc.exit:
+ orc.proc = nil
+ return nil
+
+ case <-time.After(10 * time.Second):
+ _ = orc.proc.Process.Kill()
+ orc.proc = nil
+ return <-orc.exit
+ }
+}