Merge branch 'master' into calc_snapshot_size

ci/pull_e2e_kind.groovy

@@ -132,12 +132,12 @@ e2ePodResources = [
  requests: [
  cpu: "3",
  memory: "6Gi",
- storage: "150Gi"
+ storage: "200Gi"
  ],
  limits: [
  cpu: "8",
  memory: "16Gi",
- storage: "150Gi"
+ storage: "200Gi"
  ],
 ]
 

pkg/apis/pingcap/v1alpha1/types.go

@@ -1382,6 +1382,10 @@ const (
  PDLeaderTransferAnnKey = "tidb.pingcap.com/pd-transfer-leader"
  // TiDBGracefulShutdownAnnKey is the annotation key to graceful shutdown tidb pod by user.
  TiDBGracefulShutdownAnnKey = "tidb.pingcap.com/tidb-graceful-shutdown"
+ // TiKVEvictLeaderExpirationTimeAnnKey is the annotation key to expire evict leader annotation. Type: time.RFC3339.
+ TiKVEvictLeaderExpirationTimeAnnKey = "tidb.pingcap.com/tikv-evict-leader-expiration-time"
+ // PDLeaderTransferExpirationTimeAnnKey is the annotation key to expire transfer leader annotation. Type: time.RFC3339.
+ PDLeaderTransferExpirationTimeAnnKey = "tidb.pingcap.com/pd-evict-leader-expiration-time"
 )
 
 // The `Value` of annotation controls the behavior when the leader count drops to zero, the valid value is one of:

pkg/controller/tidbcluster/pod_control.go

@@ -249,6 +249,10 @@ func (c *PodController) syncPDPod(ctx context.Context, pod *corev1.Pod, tc *v1al
  return reconcile.Result{}, nil
  }
 
+ if c.isEvictLeaderExpired(pod, v1alpha1.PDLeaderTransferExpirationTimeAnnKey) {
+ return reconcile.Result{}, c.cleanupLeaderEvictionAnnotations(pod, tc, []string{v1alpha1.PDLeaderTransferAnnKey, v1alpha1.PDLeaderTransferExpirationTimeAnnKey})
+ }
+
  // Check if there's any ongoing updates in PD.
  if tc.Status.PD.Phase != v1alpha1.NormalPhase {
  return reconcile.Result{RequeueAfter: RequeueInterval}, nil
@@ -294,6 +298,10 @@ func (c *PodController) syncTiKVPod(ctx context.Context, pod *corev1.Pod, tc *v1
  }
 
  if ok {
+ if c.isEvictLeaderExpired(pod, v1alpha1.TiKVEvictLeaderExpirationTimeAnnKey) {
+ return reconcile.Result{}, c.cleanupLeaderEvictionAnnotations(pod, tc, append(v1alpha1.EvictLeaderAnnKeys, v1alpha1.TiKVEvictLeaderExpirationTimeAnnKey))
+ }
+
  evictStatus := &v1alpha1.EvictLeaderStatus{
  PodCreateTime: pod.CreationTimestamp,
  BeginTime: metav1.Now(),
@@ -458,6 +466,34 @@ func (c *PodController) syncTiDBPod(ctx context.Context, pod *corev1.Pod, tc *v1
  return reconcile.Result{}, nil
 }
 
+func (c *PodController) cleanupLeaderEvictionAnnotations(pod *corev1.Pod, tc *v1alpha1.TidbCluster, annKeys []string) error {
+ for _, ann := range annKeys {
+ delete(pod.Annotations, ann)
+ }
+
+ if _, err := c.deps.PodControl.UpdatePod(tc, pod); err != nil {
+ return perrors.Annotatef(err, "failed delete expired annotations for tc %s/%s", pod.Namespace, pod.Name)
+ }
+ return nil
+}
+
+func (c *PodController) isEvictLeaderExpired(pod *corev1.Pod, annKey string) bool {
+ timeToExpireAnnValue, exist := pod.Annotations[annKey]
+ if exist {
+ evictionExpirationTime, err := time.Parse(time.RFC3339, timeToExpireAnnValue)
+ if err == nil {
+ if metav1.Now().Time.After(evictionExpirationTime) {
+ klog.Infof("Annotation to evict leader on the Pod %s/%s is expired", pod.Namespace, pod.Name)
+ return true
+ }
+ } else {
+ klog.Warningf("Can't parse %s value %s on %s/%s. Mark pod as expired right away. Err: ", annKey, evictionExpirationTime, pod.Namespace, pod.Name, err)
+ return true
+ }
+ }
+ return false
+}
+
 func needDeleteTiDBPod(pod *corev1.Pod) string {
  if pod.Annotations == nil {
  return ""

pkg/controller/tidbcluster/pod_control_test.go

@@ -117,7 +117,21 @@ func TestTiKVPodSync(t *testing.T) {
  if pod.Annotations == nil {
  pod.Annotations = make(map[string]string)
  }
+
+ // add expired annotation
+ pod.Annotations[v1alpha1.EvictLeaderAnnKey] = v1alpha1.EvictLeaderValueDeletePod
+ pod.Annotations[v1alpha1.TiKVEvictLeaderExpirationTimeAnnKey] = metav1.Now().Time.Add(-time.Minute).UTC().Format(time.RFC3339)
+ pod, err = deps.KubeClientset.CoreV1().Pods(pod.Namespace).Update(ctx, pod, metav1.UpdateOptions{})
+ g.Expect(err).Should(Succeed())
+
+ g.Consistently(func() int {
+ stat := c.getPodStat(pod)
+ return stat.observeAnnotationCounts
+ }, time.Second*5, interval).Should(Equal(0), "should not observe pod annotation as it is added expired")
+
+ // add annotation w/o expiration
  pod.Annotations[v1alpha1.EvictLeaderAnnKey] = v1alpha1.EvictLeaderValueDeletePod
+ delete(pod.Annotations, v1alpha1.TiKVEvictLeaderExpirationTimeAnnKey)
  pod, err = deps.KubeClientset.CoreV1().Pods(pod.Namespace).Update(ctx, pod, metav1.UpdateOptions{})
  g.Expect(err).Should(Succeed())
 
@@ -165,6 +179,7 @@ func TestPDPodSync(t *testing.T) {
  target int
  deleteAfterTransfer bool
  shouldTransfer bool
+ expirationTime time.Time
  }{
  {
  name: "transfer leader only",
@@ -216,6 +231,38 @@ func TestPDPodSync(t *testing.T) {
  deleteAfterTransfer: true,
  shouldTransfer: false,
  },
+ {
+ name: "no expiration annotation",
+ replicas: 3,
+ phase: v1alpha1.NormalPhase,
+ leader: 0,
+ failed: nil,
+ target: 0,
+ deleteAfterTransfer: true,
+ shouldTransfer: true,
+ },
+ {
+ name: "expiration annotation in the future",
+ replicas: 3,
+ phase: v1alpha1.NormalPhase,
+ leader: 0,
+ failed: nil,
+ target: 0,
+ deleteAfterTransfer: true,
+ shouldTransfer: true,
+ expirationTime: metav1.Now().Time.Add(time.Hour),
+ },
+ {
+ name: "expiration annotation in the past",
+ replicas: 3,
+ phase: v1alpha1.NormalPhase,
+ leader: 0,
+ failed: nil,
+ target: 0,
+ deleteAfterTransfer: true,
+ shouldTransfer: false,
+ expirationTime: metav1.Now().Time.Add(-time.Minute),
+ },
  }
 
  for _, c := range testCases {
@@ -307,6 +354,10 @@ func TestPDPodSync(t *testing.T) {
  } else {
  pod.Annotations[v1alpha1.PDLeaderTransferAnnKey] = v1alpha1.TransferLeaderValueNone
  }
+ if !c.expirationTime.IsZero() {
+ pod.Annotations[v1alpha1.PDLeaderTransferExpirationTimeAnnKey] = c.expirationTime.Format(time.RFC3339)
+ }
+
  pod, err = deps.KubeClientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{})
  g.Expect(err).NotTo(HaveOccurred())