check in e2e

debug in ci debug ci fix e2e checking add log E2e CI job use memory storage for etcd (pingcap#2633) * support memory storage for etcd in kind * export ETCD_STORAGE_TYPE=memory in e2e_kind.groovy * fix a bug * use KIND_ETCD_DATADIR instead of ETCD_STORAGE_TYPE * fix typos * fix bugs * Revert "fix bugs" This reverts commit 3ae083b. * revert format and simplified extraMounts Hide datasource information for backup job (pingcap#2652) Co-authored-by: pingcap-github-bot <sre-bot@pingcap.com> use crd Update pingcap_tidb_operator_build_kind.groovy
Yisaer · Jun 9, 2020 · 314cd06 · 314cd06
1 parent 79d488c
commit 314cd06
Show file tree

Hide file tree

Showing 4 changed files with 97 additions and 22 deletions.
diff --git a/ci/pingcap_tidb_operator_build_kind.groovy b/ci/pingcap_tidb_operator_build_kind.groovy
@@ -63,6 +63,9 @@ spec:
     # legacy docker path for cr.io/k8s-testimages/kubekins-e2e
     - name: docker-graph
       mountPath: /docker-graph
+    # use memory storage for etcd hostpath in kind cluster
+    - name: etcd-data-dir
+      mountPath: /mnt/tmpfs/etcd
   volumes:
   - name: modules
     hostPath:
@@ -76,6 +79,9 @@ spec:
     emptyDir: {}
   - name: docker-graph
     emptyDir: {}
+  - name: etcd-data-dir
+    emptyDir:
+      medium: Memory
   tolerations:
   - effect: NoSchedule
     key: tidb-operator
@@ -301,7 +307,7 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) {
 		}
 		}
 
-		def GLOBALS = "SKIP_BUILD=y SKIP_IMAGE_BUILD=y DOCKER_REPO=hub.pingcap.net/tidb-operator-e2e IMAGE_TAG=${GITHASH} DELETE_NAMESPACE_ON_FAILURE=true GINKGO_NO_COLOR=y"
+		def GLOBALS = "KIND_ETCD_DATADIR=/mnt/tmpfs/etcd SKIP_BUILD=y SKIP_IMAGE_BUILD=y DOCKER_REPO=hub.pingcap.net/tidb-operator-e2e IMAGE_TAG=${GITHASH} DELETE_NAMESPACE_ON_FAILURE=true GINKGO_NO_COLOR=y"
 		def builds = [:]
 		// We must not enable operator killer for Kubernetes before 1.15 in
 		// which webhook configuration does not support objectSelector. Webhook

diff --git a/cmd/backup-manager/app/util/util.go b/cmd/backup-manager/app/util/util.go
@@ -88,11 +88,11 @@ func EnsureDirectoryExist(dirName string) error {
 func OpenDB(dsn string) (*sql.DB, error) {
 	db, err := sql.Open("mysql", dsn)
 	if err != nil {
-		return nil, fmt.Errorf("open dsn %s failed, err: %v", dsn, err)
+		return nil, fmt.Errorf("open datasource failed, err: %v", err)
 	}
 	if err := db.Ping(); err != nil {
 		db.Close()
-		return nil, fmt.Errorf("cannot connect to mysql: %s, err: %v", dsn, err)
+		return nil, fmt.Errorf("cannot connect to mysql, err: %v", err)
 	}
 	return db, nil
 }

diff --git a/hack/e2e.sh b/hack/e2e.sh
@@ -64,6 +64,7 @@ Environments:
     GCR_IO_MIRROR         configure mirror for gcr.io
     QUAY_IO_MIRROR        configure mirror for quay.io
     KIND_DATA_HOSTPATH    (kind only) the host path of data directory for kind cluster, defaults: none
+    KIND_ETCD_DATADIR     (kind only) the host path of etcd data directory for kind cluster, defaults: none
     GCP_PROJECT           (gke only) the GCP project to run in
     GCP_CREDENTIALS       (gke only) the GCP service account to use
     GCP_REGION            (gke only) the GCP region, if specified a regional cluster is creaetd
@@ -187,6 +188,7 @@ SKIP_TEST=${SKIP_TEST:-}
 SKIP_DUMP=${SKIP_DUMP:-}
 REUSE_CLUSTER=${REUSE_CLUSTER:-}
 KIND_DATA_HOSTPATH=${KIND_DATA_HOSTPATH:-none}
+KIND_ETCD_DATADIR=${KIND_ETCD_DATADIR:-none}
 GCP_PROJECT=${GCP_PROJECT:-}
 GCP_CREDENTIALS=${GCP_CREDENTIALS:-}
 GCP_REGION=${GCP_REGION:-}
@@ -219,6 +221,7 @@ echo "SKIP_DOWN: $SKIP_DOWN"
 echo "SKIP_TEST: $SKIP_TEST"
 echo "SKIP_DUMP: $SKIP_DUMP"
 echo "KIND_DATA_HOSTPATH: $KIND_DATA_HOSTPATH"
+echo "KIND_ETCD_DATADIR: $KIND_ETCD_DATADIR"
 echo "GCP_PROJECT: $GCP_PROJECT"
 echo "GCP_CREDENTIALS: $GCP_CREDENTIALS"
 echo "GCP_REGION: $GCP_REGION"
@@ -366,19 +369,33 @@ EOF
 nodes:
 - role: control-plane
 EOF
-    if [[ "$KIND_DATA_HOSTPATH" != "none" ]]; then
-        if [ ! -d "$KIND_DATA_HOSTPATH" ]; then
-            echo "error: '$KIND_DATA_HOSTPATH' is not a directory"
-            exit 1
-        fi
-        local hostWorkerPath="${KIND_DATA_HOSTPATH}/control-plane"
-        test -d $hostWorkerPath || mkdir $hostWorkerPath
+    if [[ "$KIND_DATA_HOSTPATH" != "none" || "$KIND_ETCD_DATADIR" != "none" ]]; then
         cat <<EOF >> $tmpfile
   extraMounts:
+EOF
+        if [[ "$KIND_DATA_HOSTPATH" != "none" ]]; then
+            if [ ! -d "$KIND_DATA_HOSTPATH" ]; then
+                echo "error: '$KIND_DATA_HOSTPATH' is not a directory"
+                exit 1
+            fi
+            local hostWorkerPath="${KIND_DATA_HOSTPATH}/control-plane"
+            test -d $hostWorkerPath || mkdir $hostWorkerPath
+            cat <<EOF >> $tmpfile
   - containerPath: /mnt/disks/
     hostPath: "$hostWorkerPath"
     propagation: HostToContainer
 EOF
+        fi
+        if [[ "$KIND_ETCD_DATADIR" != "none" ]]; then
+            if [ ! -d "$KIND_ETCD_DATADIR" ]; then
+                echo "error: '$KIND_ETCD_DATADIR' is not a directory"
+                exit 1
+            fi
+            cat <<EOF >> $tmpfile
+  - containerPath: /var/lib/etcd
+    hostPath: "$KIND_ETCD_DATADIR"
+EOF
+        fi
     fi
     # workers
     for ((i = 1; i <= $KUBE_WORKERS; i++)) {

diff --git a/tests/e2e/tidbcluster/serial.go b/tests/e2e/tidbcluster/serial.go
@@ -17,6 +17,7 @@ import (
 	"context"
 	"fmt"
 	_ "net/http/pprof"
+	"os/exec"
 	"strconv"
 	"strings"
 	"time"
@@ -116,7 +117,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 				TestMode:          true,
 				WebhookEnabled:    true,
 				PodWebhookEnabled: true,
-				StsWebhookEnabled: false,
+				StsWebhookEnabled: true,
 			}
 			oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
 			ginkgo.By("Installing CRDs")
@@ -137,19 +138,58 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 		ginkgo.It("[PodAdmissionWebhook] able to upgrade TiDB Cluster with pod admission webhook", func() {
 			klog.Info("start to upgrade tidbcluster with pod admission webhook")
 			// deploy new cluster and test upgrade and scale-in/out with pod admission webhook
-			cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "admission", "", utilimage.TiDBV3Version)
-			cluster.Resources["pd.replicas"] = "3"
-			cluster.Resources["tikv.replicas"] = "3"
-			cluster.Resources["tidb.replicas"] = "2"
+			tc := fixture.GetTidbCluster(ns, "admission", utilimage.TiDBV3Version)
+			tc.Spec.PD.Replicas = 3
+			tc.Spec.TiKV.Replicas = 3
+			tc.Spec.TiDB.Replicas = 2
+			tests.CreateTidbClusterOrDie(cli, tc)
+			err := oa.WaitForTidbClusterReady(tc, 30*time.Minute, 5*time.Second)
+			framework.ExpectNoError(err)
+			ginkgo.By(fmt.Sprintf("Upgrading tidb cluster from %s to %s", utilimage.TiDBV3Version, utilimage.TiDBV3UpgradeVersion))
+			err = setPartitionAnnotation(ns, tc.Name, label.TiKVLabelVal, 1)
+			framework.ExpectNoError(err, "set tikv Partition annotation failed")
+			framework.Logf("set tikv Partition annotation")
+			tc, err = cli.PingcapV1alpha1().TidbClusters(ns).Get(tc.Name, metav1.GetOptions{})
+			framework.ExpectNoError(err)
+			tc.Spec.Version = utilimage.TiDBV3UpgradeVersion
+			_, err = cli.PingcapV1alpha1().TidbClusters(ns).Update(tc)
+			framework.ExpectNoError(err)
 
-			oa.DeployTidbClusterOrDie(&cluster)
-			oa.CheckTidbClusterStatusOrDie(&cluster)
-			ginkgo.By(fmt.Sprintf("Upgrading tidb cluster from %s to %s", cluster.ClusterVersion, utilimage.TiDBV3UpgradeVersion))
-			cluster.UpgradeAll(utilimage.TiDBV3UpgradeVersion)
-			oa.UpgradeTidbClusterOrDie(&cluster)
+			err = wait.Poll(5*time.Second, 30*time.Minute, func() (done bool, err error) {
+				tikvPod, err := c.CoreV1().Pods(ns).Get(fmt.Sprintf("%s-tikv-1", tc.Name), metav1.GetOptions{})
+				if err != nil {
+					return false, nil
+				}
+				if tikvPod.Spec.Containers[0].Image != fmt.Sprintf("pingcap/tikv:%s", utilimage.TiDBV3UpgradeVersion) {
+					return false, nil
+				}
+				return true, nil
+			})
+			framework.ExpectNoError(err, "expect tikv upgrade to tikv-1")
+			framework.Logf("tikv have upgraded to tikv-1")
+			err = wait.Poll(5*time.Second, 3*time.Minute, func() (done bool, err error) {
+				tikvSts, err := c.AppsV1().StatefulSets(ns).Get(fmt.Sprintf("%s-tikv", tc.Name), metav1.GetOptions{})
+				if err != nil {
+					return false, err
+				}
+				strategy := tikvSts.Spec.UpdateStrategy
+				if strategy.RollingUpdate != nil && strategy.RollingUpdate.Partition != nil &&
+					*strategy.RollingUpdate.Partition == int32(1) {
+					return false, nil
+				}
+				return true, nil
+			})
+			framework.ExpectEqual(err, wait.ErrWaitTimeout, "tikv Partition should remain 1 for 3 min")
+			framework.Logf("tikv Partition remain 1 for 3 min")
+			tc, err = cli.PingcapV1alpha1().TidbClusters(ns).Get(tc.Name, metav1.GetOptions{})
+			framework.ExpectNoError(err)
+			tc.Annotations = nil
+			_, err = cli.PingcapV1alpha1().TidbClusters(ns).Update(tc)
+			framework.ExpectNoError(err)
+			framework.Logf("tidbcluster annotation have been cleaned")
 			// TODO: find a more graceful way to check tidbcluster during upgrading
-			oa.CheckTidbClusterStatusOrDie(&cluster)
-			oa.CleanTidbClusterOrDie(&cluster)
+			err = oa.WaitForTidbClusterReady(tc, 30*time.Minute, 5*time.Second)
+			framework.ExpectNoError(err)
 		})
 	})
 
@@ -912,3 +952,15 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 		})
 	})
 })
+
+func setPartitionAnnotation(namespace, tcName, component string, ordinal int) error {
+	// add annotation to pause statefulset upgrade process
+	cmd := fmt.Sprintf("kubectl annotate tc %s -n %s tidb.pingcap.com/%s-partition=%d --overwrite",
+		tcName, namespace, component, ordinal)
+	klog.Infof("%s", cmd)
+	output, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("fail to set annotation for [%s/%s], component: %s, partition: %d, err: %v, output: %s", namespace, tcName, component, ordinal, err, string(output))
+	}
+	return nil
+}