ttl: make ttl manually trigger stable (#45869) (#45896)

close #45868

ttl/client/command.go

@@ -29,7 +29,7 @@ import (
 )
 
 const (
- ttlCmdKeyLeaseSeconds int64 = 60
+ ttlCmdKeyLeaseSeconds int64 = 180
  ttlCmdKeyRequestPrefix = "/tidb/ttl/cmd/req/"
  ttlCmdKeyResponsePrefix = "/tidb/ttl/cmd/resp/"
  ttlCmdTypeTriggerTTLJob = "trigger_ttl_job"
@@ -161,13 +161,15 @@ func (c *etcdClient) waitCmdResponse(ctx context.Context, reqID string, obj inte
 
  key := ttlCmdKeyResponsePrefix + reqID
  ch := c.etcdCli.Watch(ctx, key)
- ticker := time.NewTimer(time.Second)
+ ticker := time.NewTicker(time.Second)
  defer ticker.Stop()
 
  var respData []byte
 loop:
  for {
  select {
+ case <-ctx.Done():
+ return ctx.Err()
  case <-ticker.C:
  response, err := c.etcdCli.Get(ctx, key)
  if err != nil {
@@ -178,7 +180,15 @@ loop:
  respData = response.Kvs[0].Value
  break loop
  }
- case resp := <-ch:
+ case resp, ok := <-ch:
+ if !ok {
+ logutil.BgLogger().Info("watcher is closed")
+ // to make ch always block
+ ch = make(chan clientv3.WatchResponse)
+ time.Sleep(time.Second)
+ continue loop
+ }
+
  for _, event := range resp.Events {
  if event.Type == clientv3.EventTypePut {
  respData = event.Kv.Value