From d207824d79b0a78579e463fbb64b75dd135af963 Mon Sep 17 00:00:00 2001
From: Ti Chi Robot <ti-community-prow-bot@tidb.io>
Date: Thu, 4 May 2023 14:33:55 +0800
Subject: [PATCH] *: provide a option to wait for init stats to finish before
 providing service during startup (#43381) (#43447)

ref pingcap/tidb#42160, close pingcap/tidb#43385
---
 config/config.go            |  6 ++++++
 config/config.toml.example  |  3 +++
 config/config_test.go       |  2 ++
 domain/domain.go            | 33 +++++++++++++++++++++------------
 statistics/handle/handle.go |  3 +++
 tidb-server/main.go         |  3 +++
 6 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/config/config.go b/config/config.go
index eccaf1e7b3e17..63c181d51675e 100644
--- a/config/config.go
+++ b/config/config.go
@@ -719,6 +719,11 @@ type Performance struct {
 	EnableLoadFMSketch bool `toml:"enable-load-fmsketch" json:"enable-load-fmsketch"`
 
 	LiteInitStats bool `toml:"lite-init-stats" json:"lite-init-stats"`
+
+	// If ForceInitStats is true, when tidb starts up, it doesn't provide service until init stats is finished.
+	// If ForceInitStats is false, tidb can provide service before init stats is finished. Note that during the period
+	// of init stats the optimizer may make bad decisions due to pseudo stats.
+	ForceInitStats bool `toml:"force-init-stats" json:"force-init-stats"`
 }
 
 // PlanCache is the PlanCache section of the config.
@@ -985,6 +990,7 @@ var defaultConf = Config{
 		RunAutoAnalyze:                    true,
 		EnableLoadFMSketch:                false,
 		LiteInitStats:                     false,
+		ForceInitStats:                    false,
 	},
 	ProxyProtocol: ProxyProtocol{
 		Networks:      "",
diff --git a/config/config.toml.example b/config/config.toml.example
index 74c6623a023ad..18c7de3088916 100644
--- a/config/config.toml.example
+++ b/config/config.toml.example
@@ -282,6 +282,9 @@ gogc = 100
 # Whether to use the lite mode of init stats.
 lite-init-stats = false
 
+# Whether to wait for init stats to finish before providing service during startup
+force-init-stats = false
+
 [proxy-protocol]
 # PROXY protocol acceptable client networks.
 # Empty string means disable PROXY protocol, * means all networks.
diff --git a/config/config_test.go b/config/config_test.go
index 8cf8766075f98..5523165b935d9 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -742,6 +742,7 @@ tcp-no-delay = false
 enable-load-fmsketch = true
 plan-replayer-dump-worker-concurrency = 1
 lite-init-stats = true
+force-init-stats = true
 [tikv-client]
 commit-timeout="41s"
 max-batch-size=128
@@ -834,6 +835,7 @@ max_connections = 200
 	require.Equal(t, 40960, conf.Status.GRPCMaxSendMsgSize)
 	require.True(t, conf.Performance.EnableLoadFMSketch)
 	require.True(t, conf.Performance.LiteInitStats)
+	require.True(t, conf.Performance.ForceInitStats)
 
 	err = f.Truncate(0)
 	require.NoError(t, err)
diff --git a/domain/domain.go b/domain/domain.go
index 88e56847f20c2..4914e90158b1d 100644
--- a/domain/domain.go
+++ b/domain/domain.go
@@ -2149,20 +2149,11 @@ func (do *Domain) newOwnerManager(prompt, ownerKey string) owner.Manager {
 	return statsOwner
 }
 
-func (do *Domain) loadStatsWorker() {
-	defer util.Recover(metrics.LabelDomain, "loadStatsWorker", nil, false)
-	lease := do.statsLease
-	if lease == 0 {
-		lease = 3 * time.Second
-	}
-	loadTicker := time.NewTicker(lease)
-	updStatsHealthyTicker := time.NewTicker(20 * lease)
+func (do *Domain) initStats() {
+	statsHandle := do.StatsHandle()
 	defer func() {
-		loadTicker.Stop()
-		updStatsHealthyTicker.Stop()
-		logutil.BgLogger().Info("loadStatsWorker exited.")
+		close(statsHandle.InitStatsDone)
 	}()
-	statsHandle := do.StatsHandle()
 	t := time.Now()
 	liteInitStats := config.GetGlobalConfig().Performance.LiteInitStats
 	var err error
@@ -2176,6 +2167,24 @@ func (do *Domain) loadStatsWorker() {
 	} else {
 		logutil.BgLogger().Info("init stats info time", zap.Bool("lite", liteInitStats), zap.Duration("take time", time.Since(t)))
 	}
+}
+
+func (do *Domain) loadStatsWorker() {
+	defer util.Recover(metrics.LabelDomain, "loadStatsWorker", nil, false)
+	lease := do.statsLease
+	if lease == 0 {
+		lease = 3 * time.Second
+	}
+	loadTicker := time.NewTicker(lease)
+	updStatsHealthyTicker := time.NewTicker(20 * lease)
+	defer func() {
+		loadTicker.Stop()
+		updStatsHealthyTicker.Stop()
+		logutil.BgLogger().Info("loadStatsWorker exited.")
+	}()
+	do.initStats()
+	statsHandle := do.StatsHandle()
+	var err error
 	for {
 		select {
 		case <-loadTicker.C:
diff --git a/statistics/handle/handle.go b/statistics/handle/handle.go
index 6250b6a5f6dc6..ce340bcffe715 100644
--- a/statistics/handle/handle.go
+++ b/statistics/handle/handle.go
@@ -128,6 +128,8 @@ type Handle struct {
 	serverIDGetter func() uint64
 	// tableLocked used to store locked tables
 	tableLocked []int64
+
+	InitStatsDone chan struct{}
 }
 
 // GetTableLockedAndClearForTest for unit test only
@@ -483,6 +485,7 @@ func NewHandle(ctx, initStatsCtx sessionctx.Context, lease time.Duration, pool s
 		pool:             pool,
 		sysProcTracker:   tracker,
 		serverIDGetter:   serverIDGetter,
+		InitStatsDone:    make(chan struct{}),
 	}
 	handle.initStatsCtx = initStatsCtx
 	handle.lease.Store(lease)
diff --git a/tidb-server/main.go b/tidb-server/main.go
index c1f1109c35499..ff5e0e7b5cce8 100644
--- a/tidb-server/main.go
+++ b/tidb-server/main.go
@@ -259,6 +259,9 @@ func main() {
 		close(exited)
 	})
 	topsql.SetupTopSQL()
+	if config.GetGlobalConfig().Performance.ForceInitStats {
+		<-dom.StatsHandle().InitStatsDone
+	}
 	terror.MustNil(svr.Run())
 	<-exited
 	syncLog()