rewrite e2e (#346)

* rewrite e2e

.gitignore

@@ -3,6 +3,8 @@ images/tidb-operator/bin/
 images/tidb-operator-e2e/bin/
 images/tidb-operator-e2e/tidb-cluster/
 images/tidb-operator-e2e/tidb-operator/
+tests/images/e2e/tidb-cluster/
+tests/images/e2e/tidb-operator/
 *.tar
 tmp/
 data/

Makefile

@@ -42,10 +42,16 @@ e2e-docker-push: e2e-docker
  docker push "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:latest"
 
 e2e-docker: e2e-build
+ [[ -d tests/images/e2e/tidb-operator ]] && rm -r tests/images/e2e/tidb-operator || true
+ [[ -d tests/images/e2e/tidb-cluster ]] && rm -r tests/images/e2e/tidb-cluster || true
+ [[ -d tests/images/e2e/tidb-backup ]] && rm -r tests/images/e2e/tidb-backup || true
+ cp -r charts/tidb-operator tests/images/e2e
+ cp -r charts/tidb-cluster tests/images/e2e
+ cp -r charts/tidb-backup tests/images/e2e
  docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:latest" tests/images/e2e
 
 e2e-build:
- $(GOENV) ginkgo build tests/e2e
+ $(GO) -ldflags '$(LDFLAGS)' -o tests/images/e2e/bin/e2e tests/cmd/e2e/main.go
 
 test:
  @echo "Run unit tests"

tests/actions.go

@@ -116,6 +116,7 @@ type OperatorInfo struct {
  Image string
  Tag string
  SchedulerImage string
+ SchedulerTag string
  LogLevel string
 }
 
@@ -196,28 +197,43 @@ func (tc *TidbClusterInfo) TidbClusterHelmSetString(m map[string]string) string
  return strings.Join(arr, ",")
 }
 
-func (oa *operatorActions) DeployOperator(info *OperatorInfo) error {
- if err := cloneOperatorRepo(); err != nil {
- return err
+func (oi *OperatorInfo) OperatorHelmSetString(m map[string]string) string {
+ set := map[string]string{
+ "operatorImage": oi.Image,
+ "controllerManager.autoFailover": "true",
+ "scheduler.kubeSchedulerImageName": oi.SchedulerImage,
+ "controllerManager.logLevel": oi.LogLevel,
+ "scheduler.logLevel": "2",
  }
- if err := checkoutTag(info.Tag); err != nil {
- return err
+ if oi.SchedulerTag != "" {
+ set["scheduler.kubeSchedulerImageTag"] = oi.SchedulerTag
+ }
+
+ arr := make([]string, 0, len(set))
+ for k, v := range set {
+ arr = append(arr, fmt.Sprintf("%s=%s", k, v))
+ }
+ return strings.Join(arr, ",")
+}
+
+func (oa *operatorActions) DeployOperator(info *OperatorInfo) error {
+ if info.Tag != "e2e" {
+ if err := cloneOperatorRepo(); err != nil {
+ return err
+ }
+ if err := checkoutTag(info.Tag); err != nil {
+ return err
+ }
  }
 
  cmd := fmt.Sprintf(`helm install /charts/%s/tidb-operator \
  --name %s \
  --namespace %s \
- --set operatorImage=%s \
- --set controllerManager.autoFailover=true \
- --set scheduler.kubeSchedulerImage=%s \
- --set controllerManager.logLevel=%s \
- --set scheduler.logLevel=2`,
+ --set-string %s`,
  info.Tag,
  info.ReleaseName,
  info.Namespace,
- info.Image,
- info.SchedulerImage,
- info.LogLevel)
+ info.OperatorHelmSetString(nil))
  glog.Info(cmd)
  res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
  if err != nil {
@@ -257,7 +273,17 @@ func (oa *operatorActions) DeployTidbCluster(info *TidbClusterInfo) error {
  glog.Infof("deploy tidb cluster end cluster[%s] namespace[%s]", info.ClusterName, info.Namespace)
  }()
 
- err := oa.CreateSecret(info)
+ namespace := &corev1.Namespace{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: info.Namespace,
+ },
+ }
+ _, err := oa.kubeCli.CoreV1().Namespaces().Create(namespace)
+ if err != nil && !errors.IsAlreadyExists(err) {
+ return fmt.Errorf("failed to create namespace[%s]:%v", info.Namespace, err)
+ }
+
+ err = oa.CreateSecret(info)
  if err != nil {
  return fmt.Errorf("failed to create secret of cluster [%s]: %v", info.ClusterName, err)
  }
@@ -315,6 +341,12 @@ func (oa *operatorActions) CleanTidbCluster(info *TidbClusterInfo) error {
  }
  }
 
+ // delete all jobs
+ allJobsSet := label.Label{}.Instance(info.ClusterName).String()
+ if res, err := exec.Command("kubectl", "delete", "jobs", "-n", info.Namespace, "-l", allJobsSet).CombinedOutput(); err != nil {
+ return fmt.Errorf("failed to delete jobs: %v, %s", err, string(res))
+ }
+
  patchPVCmd := fmt.Sprintf(`kubectl get pv -l %s=%s,%s=%s --output=name | xargs -I {} \
  kubectl patch {} -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'`,
  label.NamespaceLabelKey, info.Namespace, label.InstanceLabelKey, info.ClusterName)

tests/cmd/e2e/main.go

@@ -15,17 +15,15 @@ package main
 
 import (
  "fmt"
- "net/http"
  _ "net/http/pprof"
 
- "github.com/pingcap/tidb-operator/tests/pkg/workload"
- "github.com/pingcap/tidb-operator/tests/pkg/workload/ddl"
-
  "github.com/golang/glog"
  "github.com/jinzhu/copier"
  "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
  "github.com/pingcap/tidb-operator/tests"
  "github.com/pingcap/tidb-operator/tests/backup"
+ "github.com/pingcap/tidb-operator/tests/pkg/workload"
+ "github.com/pingcap/tidb-operator/tests/pkg/workload/ddl"
  "k8s.io/apiserver/pkg/util/logs"
  "k8s.io/client-go/kubernetes"
  "k8s.io/client-go/rest"
@@ -41,16 +39,6 @@ func main() {
  glog.Fatalf("failed to parse config: %v", err)
  }
 
- go func() {
- glog.Info(http.ListenAndServe("localhost:6060", nil))
- }()
-
- // TODO read these args from config
- beginTidbVersion := "v2.1.0"
- toTidbVersion := "v2.1.4"
- operatorTag := "master"
- operatorImage := "pingcap/tidb-operator:latest"
-
  cfg, err := rest.InClusterConfig()
  if err != nil {
  glog.Fatalf("failed to get config: %v", err)
@@ -69,12 +57,17 @@ func main() {
  operatorInfo := &tests.OperatorInfo{
  Namespace: "pingcap",
  ReleaseName: "operator",
- Image: operatorImage,
- Tag: operatorTag,
- SchedulerImage: "gcr.io/google-containers/hyperkube:v1.12.1",
+ Image: conf.OperatorImage,
+ Tag: conf.OperatorTag,
+ SchedulerImage: "mirantis/hypokube",
+ SchedulerTag: "final",
  LogLevel: "2",
  }
 
+ initTidbVersion, err := conf.GetInitTidbVersion()
+ if err != nil {
+ glog.Fatal(err)
+ }
  // create database and table and insert a column for test backup and restore
  initSql := `"create database record;use record;create table test(t char(32))"`
 
@@ -84,10 +77,10 @@ func main() {
  {
  Namespace: name1,
  ClusterName: name1,
- OperatorTag: operatorTag,
- PDImage: fmt.Sprintf("pingcap/pd:%s", beginTidbVersion),
- TiKVImage: fmt.Sprintf("pingcap/tikv:%s", beginTidbVersion),
- TiDBImage: fmt.Sprintf("pingcap/tidb:%s", beginTidbVersion),
+ OperatorTag: conf.OperatorTag,
+ PDImage: fmt.Sprintf("pingcap/pd:%s", initTidbVersion),
+ TiKVImage: fmt.Sprintf("pingcap/tikv:%s", initTidbVersion),
+ TiDBImage: fmt.Sprintf("pingcap/tidb:%s", initTidbVersion),
  StorageClassName: "local-storage",
  Password: "admin",
  InitSql: initSql,
@@ -115,10 +108,10 @@ func main() {
  {
  Namespace: name2,
  ClusterName: name2,
- OperatorTag: "master",
- PDImage: fmt.Sprintf("pingcap/pd:%s", beginTidbVersion),
- TiKVImage: fmt.Sprintf("pingcap/tikv:%s", beginTidbVersion),
- TiDBImage: fmt.Sprintf("pingcap/tidb:%s", beginTidbVersion),
+ OperatorTag: conf.OperatorTag,
+ PDImage: fmt.Sprintf("pingcap/pd:%s", initTidbVersion),
+ TiKVImage: fmt.Sprintf("pingcap/tikv:%s", initTidbVersion),
+ TiDBImage: fmt.Sprintf("pingcap/tidb:%s", initTidbVersion),
  StorageClassName: "local-storage",
  Password: "admin",
  InitSql: initSql,
@@ -231,26 +224,29 @@ func main() {
  }
  }
 
+ // upgrade test
+ upgradeTidbVersions := conf.GetUpgradeTidbVersions()
+ for _, upgradeTidbVersion := range upgradeTidbVersions {
+ for _, clusterInfo := range clusterInfos {
+ clusterInfo = clusterInfo.UpgradeAll(upgradeTidbVersion)
+ if err = oa.UpgradeTidbCluster(clusterInfo); err != nil {
+ glog.Fatal(err)
+ }
+ }
+ for _, clusterInfo := range clusterInfos {
+ if err = oa.CheckTidbClusterStatus(clusterInfo); err != nil {
+ glog.Fatal(err)
+ }
+ }
+ }
+
  return nil
  }, workloads...)
 
  if err != nil {
  glog.Fatal(err)
  }
 
- for _, clusterInfo := range clusterInfos {
- clusterInfo = clusterInfo.UpgradeAll(toTidbVersion)
- if err = oa.UpgradeTidbCluster(clusterInfo); err != nil {
- glog.Fatal(err)
- }
- }
-
- for _, clusterInfo := range clusterInfos {
- if err = oa.CheckTidbClusterStatus(clusterInfo); err != nil {
- glog.Fatal(err)
- }
- }
-
  // backup and restore
  backupClusterInfo := clusterInfos[0]
  restoreClusterInfo := &tests.TidbClusterInfo{}

tests/config.go

@@ -2,7 +2,9 @@ package tests
 
 import (
  "flag"
+ "fmt"
  "io/ioutil"
+ "strings"
 
  yaml "gopkg.in/yaml.v2"
 )
@@ -11,6 +13,9 @@ import (
 type Config struct {
  configFile string
 
+ TidbVersions string `yaml:"tidb_versions" json:"tidb_versions"`
+ OperatorTag string `yaml:"operator_tag" json:"operator_tag"`
+ OperatorImage string `yaml:"operator_image" json:"operator_image"`
  LogDir string `yaml:"log_dir" json:"log_dir"`
  FaultTriggerPort int `yaml:"fault_trigger_port" json:"fault_trigger_port"`
  Nodes []Nodes `yaml:"nodes" json:"nodes"`
@@ -27,9 +32,12 @@ type Nodes struct {
 // NewConfig creates a new config.
 func NewConfig() *Config {
  cfg := &Config{}
- flag.StringVar(&cfg.configFile, "config", "/etc/e2e/config.yaml", "Config file")
+ flag.StringVar(&cfg.configFile, "config", "", "Config file")
  flag.StringVar(&cfg.LogDir, "log-dir", "/logDir", "log directory")
  flag.IntVar(&cfg.FaultTriggerPort, "fault-trigger-port", 23332, "the http port of fault trigger service")
+ flag.StringVar(&cfg.TidbVersions, "tidb-versions", "v2.1.3,v2.1.4", "tidb versions")
+ flag.StringVar(&cfg.OperatorTag, "operator-tag", "master", "operator tag used to choose charts")
+ flag.StringVar(&cfg.OperatorImage, "operator-image", "pingcap/tidb-operator:latest", "operator image")
 
  return cfg
 }
@@ -63,3 +71,18 @@ func (c *Config) configFromFile(path string) error {
 
  return nil
 }
+
+func (c *Config) GetInitTidbVersion() (string, error) {
+ tidbVersions := strings.Split(c.TidbVersions, ",")
+ if len(tidbVersions) == 0 {
+ return "", fmt.Errorf("init tidb versions can not be nil")
+ }
+
+ return tidbVersions[0], nil
+}
+
+func (c *Config) GetUpgradeTidbVersions() []string {
+ tidbVersions := strings.Split(c.TidbVersions, ",")
+
+ return tidbVersions[1:]
+}

tests/images/e2e/Dockerfile

@@ -14,4 +14,8 @@ RUN curl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VER
  rm -rf linux-amd64 && \
  rm helm-${HELM_VERSION}-linux-amd64.tar.gz
 
+ADD tidb-operator /charts/e2e/tidb-operator
+ADD tidb-cluster /charts/e2e/tidb-cluster
+ADD tidb-backup /charts/e2e/tidb-backup
+
 ADD bin/e2e /usr/local/bin/e2e

tests/manifests/e2e/e2e.yaml

@@ -0,0 +1,45 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: tidb-operator-e2e
+subjects:
+- kind: ServiceAccount
+ namespace: tidb-operator-e2e
+ name: tidb-operator-e2e
+roleRef:
+ kind: ClusterRole
+ name: cluster-admin
+ apiGroup: rbac.authorization.k8s.io
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+ namespace: tidb-operator-e2e
+ name: tidb-operator-e2e
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ namespace: tidb-operator-e2e
+ name: tidb-operator-e2e
+spec:
+ serviceAccount: tidb-operator-e2e
+ containers:
+ - name: tidb-operator-e2e
+ image: ""
+ imagePullPolicy: Always
+ command:
+ - /usr/local/bin/e2e
+ - --operator-tag=e2e
+ - --operator-image=pingcap/tidb-operator:latest
+ - --tidb-versions=v2.1.3,v2.1.4
+ volumeMounts:
+ - mountPath: /logDir
+ name: logdir
+ volumes:
+ - name: logdir
+ hostPath:
+ path: /var/log
+ type: Directory
+ restartPolicy: Never

tests/manifests/stability/stability-configmap.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1 
+kind: ConfigMap
+metadata:
+ namespace: tidb-operator-e2e
+ name: tidb-operator-e2e-config
+data:
+ e2e-config: |- 
+ nodes:
+ - physical_node: 172.16.4.39
+ nodes:
+ - 172.16.4.171
+ - 172.16.4.172
+ - 172.16.4.173
+ - physical_node: 172.16.4.40
+ nodes:
+ - 172.16.4.174
+ - 172.16.4.175
+ - 172.16.4.176
+ etcds: 
+ - physical_node: 172.16.4.39
+ nodes:
+ - 172.16.4.171
+ - 172.16.4.172
+ - 172.16.4.173
+ apiservers: 
+ - physical_node: 172.16.4.39
+ nodes:
+ - 172.16.4.171
+ - 172.16.4.172
+ - 172.16.4.173
+
+