Make StsWebhook and PodWebhook work (#2664)

* remove statefulsets webhook

* fix webhook partition

* check in e2e

debug in ci

debug ci

fix e2e checking

add log

E2e CI job use memory storage for etcd (#2633)

* support memory storage for etcd in kind

* export ETCD_STORAGE_TYPE=memory in e2e_kind.groovy

* fix a bug

* use KIND_ETCD_DATADIR instead of ETCD_STORAGE_TYPE

* fix typos

* fix bugs

* Revert "fix bugs"

This reverts commit 3ae083b.

* revert format and simplified extraMounts

Hide datasource information for backup job (#2652)

Co-authored-by: pingcap-github-bot <sre-bot@pingcap.com>

use crd

Update pingcap_tidb_operator_build_kind.groovy

* use RunKubectl

Co-authored-by: pingcap-github-bot <sre-bot@pingcap.com>

pkg/manager/member/pd_upgrader.go

@@ -72,11 +72,6 @@ func (pu *pdUpgrader) gracefulUpgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Sta
 		return nil
 	}
 
-	if controller.PodWebhookEnabled {
-		setUpgradePartition(newSet, 0)
-		return nil
-	}
-
 	setUpgradePartition(newSet, *oldSet.Spec.UpdateStrategy.RollingUpdate.Partition)
 	podOrdinals := helper.GetPodOrdinals(*oldSet.Spec.Replicas, oldSet).List()
 	for _i := len(podOrdinals) - 1; _i >= 0; _i-- {
@@ -99,6 +94,11 @@ func (pu *pdUpgrader) gracefulUpgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Sta
 			continue
 		}
 
+		if controller.PodWebhookEnabled {
+			setUpgradePartition(newSet, i)
+			return nil
+		}
+
 		return pu.upgradePDPod(tc, i, newSet)
 	}
 

pkg/manager/member/tikv_upgrader.go

@@ -87,11 +87,6 @@ func (tku *tikvUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Stateful
 		return nil
 	}
 
-	if controller.PodWebhookEnabled {
-		setUpgradePartition(newSet, 0)
-		return nil
-	}
-
 	setUpgradePartition(newSet, *oldSet.Spec.UpdateStrategy.RollingUpdate.Partition)
 	podOrdinals := helper.GetPodOrdinals(*oldSet.Spec.Replicas, oldSet).List()
 	for _i := len(podOrdinals) - 1; _i >= 0; _i-- {
@@ -122,6 +117,11 @@ func (tku *tikvUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Stateful
 			continue
 		}
 
+		if controller.PodWebhookEnabled {
+			setUpgradePartition(newSet, i)
+			return nil
+		}
+
 		return tku.upgradeTiKVPod(tc, i, newSet)
 	}
 

tests/e2e/tidbcluster/serial.go

@@ -116,7 +116,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 				TestMode:          true,
 				WebhookEnabled:    true,
 				PodWebhookEnabled: true,
-				StsWebhookEnabled: false,
+				StsWebhookEnabled: true,
 			}
 			oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
 			ginkgo.By("Installing CRDs")
@@ -137,19 +137,58 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 		ginkgo.It("[PodAdmissionWebhook] able to upgrade TiDB Cluster with pod admission webhook", func() {
 			klog.Info("start to upgrade tidbcluster with pod admission webhook")
 			// deploy new cluster and test upgrade and scale-in/out with pod admission webhook
-			cluster := newTidbClusterConfig(e2econfig.TestConfig, ns, "admission", "", utilimage.TiDBV3Version)
-			cluster.Resources["pd.replicas"] = "3"
-			cluster.Resources["tikv.replicas"] = "3"
-			cluster.Resources["tidb.replicas"] = "2"
+			tc := fixture.GetTidbCluster(ns, "admission", utilimage.TiDBV3Version)
+			tc.Spec.PD.Replicas = 3
+			tc.Spec.TiKV.Replicas = 3
+			tc.Spec.TiDB.Replicas = 2
+			tests.CreateTidbClusterOrDie(cli, tc)
+			err := oa.WaitForTidbClusterReady(tc, 30*time.Minute, 5*time.Second)
+			framework.ExpectNoError(err)
+			ginkgo.By(fmt.Sprintf("Upgrading tidb cluster from %s to %s", utilimage.TiDBV3Version, utilimage.TiDBV3UpgradeVersion))
+			err = setPartitionAnnotation(ns, tc.Name, label.TiKVLabelVal, 1)
+			framework.ExpectNoError(err, "set tikv Partition annotation failed")
+			framework.Logf("set tikv Partition annotation")
+			tc, err = cli.PingcapV1alpha1().TidbClusters(ns).Get(tc.Name, metav1.GetOptions{})
+			framework.ExpectNoError(err)
+			tc.Spec.Version = utilimage.TiDBV3UpgradeVersion
+			_, err = cli.PingcapV1alpha1().TidbClusters(ns).Update(tc)
+			framework.ExpectNoError(err)
 
-			oa.DeployTidbClusterOrDie(&cluster)
-			oa.CheckTidbClusterStatusOrDie(&cluster)
-			ginkgo.By(fmt.Sprintf("Upgrading tidb cluster from %s to %s", cluster.ClusterVersion, utilimage.TiDBV3UpgradeVersion))
-			cluster.UpgradeAll(utilimage.TiDBV3UpgradeVersion)
-			oa.UpgradeTidbClusterOrDie(&cluster)
+			err = wait.Poll(5*time.Second, 30*time.Minute, func() (done bool, err error) {
+				tikvPod, err := c.CoreV1().Pods(ns).Get(fmt.Sprintf("%s-tikv-1", tc.Name), metav1.GetOptions{})
+				if err != nil {
+					return false, nil
+				}
+				if tikvPod.Spec.Containers[0].Image != fmt.Sprintf("pingcap/tikv:%s", utilimage.TiDBV3UpgradeVersion) {
+					return false, nil
+				}
+				return true, nil
+			})
+			framework.ExpectNoError(err, "expect tikv upgrade to tikv-1")
+			framework.Logf("tikv have upgraded to tikv-1")
+			err = wait.Poll(5*time.Second, 3*time.Minute, func() (done bool, err error) {
+				tikvSts, err := c.AppsV1().StatefulSets(ns).Get(fmt.Sprintf("%s-tikv", tc.Name), metav1.GetOptions{})
+				if err != nil {
+					return false, err
+				}
+				strategy := tikvSts.Spec.UpdateStrategy
+				if strategy.RollingUpdate != nil && strategy.RollingUpdate.Partition != nil &&
+					*strategy.RollingUpdate.Partition == int32(1) {
+					return false, nil
+				}
+				return true, nil
+			})
+			framework.ExpectEqual(err, wait.ErrWaitTimeout, "tikv Partition should remain 1 for 3 min")
+			framework.Logf("tikv Partition remain 1 for 3 min")
+			tc, err = cli.PingcapV1alpha1().TidbClusters(ns).Get(tc.Name, metav1.GetOptions{})
+			framework.ExpectNoError(err)
+			tc.Annotations = nil
+			_, err = cli.PingcapV1alpha1().TidbClusters(ns).Update(tc)
+			framework.ExpectNoError(err)
+			framework.Logf("tidbcluster annotation have been cleaned")
 			// TODO: find a more graceful way to check tidbcluster during upgrading
-			oa.CheckTidbClusterStatusOrDie(&cluster)
-			oa.CleanTidbClusterOrDie(&cluster)
+			err = oa.WaitForTidbClusterReady(tc, 30*time.Minute, 5*time.Second)
+			framework.ExpectNoError(err)
 		})
 	})
 
@@ -952,3 +991,12 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 		})
 	})
 })
+
+func setPartitionAnnotation(namespace, tcName, component string, ordinal int) error {
+	// add annotation to pause statefulset upgrade process
+	output, err := framework.RunKubectl("annotate", "tc", tcName, "-n", namespace, fmt.Sprintf("tidb.pingcap.com/%s-partition=%d", component, ordinal), "--overwrite")
+	if err != nil {
+		return fmt.Errorf("fail to set annotation for [%s/%s], component: %s, partition: %d, err: %v, output: %s", namespace, tcName, component, ordinal, err, output)
+	}
+	return nil
+}