Sync store may block the process of leader exit

[rleungx]

Bug Report

What did you do?

Transfer PD leader

What did you expect to see?

It should be finished in a short time.

What did you see instead?

Cost about 3 min to finish

It will try to connect to the tombstone store and doesn't have any timeout mechanism. And the store list is only updated when fail to request all stores. So If we offline a store, even if we do remove-tombstone, it could still be on the list.

// runSyncConfig runs the job to sync tikv config.
func (c *RaftCluster) runSyncConfig() {
	defer logutil.LogPanic()
	defer c.wg.Done()

	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()
	stores := c.GetStores()

	syncConfig(c.storeConfigManager, stores)
	for {
		select {
		case <-c.ctx.Done():
			log.Info("sync store config job is stopped")
			return
		case <-ticker.C:
			if !syncConfig(c.storeConfigManager, stores) {
				stores = c.GetStores() // update until all stores fail 
			}
		}
	}
}

func syncConfig(manager *config.StoreConfigManager, stores []*core.StoreInfo) bool {
	for index := 0; index < len(stores); index++ {
		// filter out the stores that are tiflash
		store := stores[index]
		if core.IsStoreContainLabel(store.GetMeta(), core.EngineKey, core.EngineTiFlash) {
			continue
		}

		// filter out the stores that are not up.
		if !(store.IsPreparing() || store.IsServing()) {
			continue
		}
		// it will try next store if the current store is failed.
		address := netutil.ResolveLoopBackAddr(stores[index].GetStatusAddress(), stores[index].GetAddress())
		if err := manager.ObserveConfig(address); err != nil {
			storeSyncConfigEvent.WithLabelValues(address, "fail").Inc()
			log.Debug("sync store config failed, it will try next store", zap.Error(err))
			continue
		}
		storeSyncConfigEvent.WithLabelValues(address, "succ").Inc()
		// it will only try one store.
		return true
	}
	return false
}

// GetConfig returns the store config from TiKV.
func (s TiKVConfigSource) GetConfig(statusAddress string) (*StoreConfig, error) {
	url := fmt.Sprintf("%s://%s/config", s.schema, statusAddress)
	resp, err := s.client.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var cfg StoreConfig
	if err := json.Unmarshal(body, &cfg); err != nil {
		return nil, err
	}
	return &cfg, nil
}

What version of PD are you using (pd-server -V)?

v6.1.0

summary:

fixed in :

• v6.5.4: config: sync store config in time (#6919) #6932
• v6.1.0 config: sync store config in time (#6919) #6931
• v7.1.2 config: sync store config in time (#6919) #6933