server(ticdc): use TiDB gcTuner in a more reasonable way (#9786) (#9968)

close #9762

cdc/server/server.go

@@ -43,7 +43,6 @@ import (
 	"github.com/pingcap/tiflow/pkg/p2p"
 	"github.com/pingcap/tiflow/pkg/pdutil"
 	"github.com/pingcap/tiflow/pkg/tcpserver"
-	"github.com/pingcap/tiflow/pkg/util"
 	p2pProto "github.com/pingcap/tiflow/proto/p2p"
 	pd "github.com/tikv/pd/client"
 	"go.uber.org/zap"
@@ -61,6 +60,8 @@ const (
 	maxHTTPConnection = 1000
 	// httpConnectionTimeout is used to limit a connection max alive time of http server.
 	httpConnectionTimeout = 10 * time.Minute
+	// maxGcTunerMemory is used to limit the max memory usage of cdc server. if the memory is larger than it, gc tuner will be disabled
+	maxGcTunerMemory = 512 * 1024 * 1024 * 1024
 )
 
 // Server is the interface for the TiCDC server
@@ -208,34 +209,33 @@ func (s *server) prepare(ctx context.Context) error {
 	if err := s.startActorSystems(ctx); err != nil {
 		return errors.Trace(err)
 	}
-
-	if err := s.setMemoryLimit(); err != nil {
-		return errors.Trace(err)
-	}
-
+	s.setMemoryLimit()
 	s.capture = capture.NewCapture(
 		s.pdEndpoints, cdcEtcdClient, s.grpcService,
 		s.tableActorSystem, s.sortEngineFactory, s.sorterSystem, s.pdClient)
 
 	return nil
 }
 
-func (s *server) setMemoryLimit() error {
+func (s *server) setMemoryLimit() {
 	conf := config.GetGlobalServerConfig()
-	totalMemory, err := util.GetMemoryLimit()
-	if err != nil {
-		return errors.Trace(err)
+	if conf.GcTunerMemoryThreshold > maxGcTunerMemory {
+		// If total memory is larger than 512GB, we will not set memory limit.
+		// Because the memory limit is not accurate, and it is not necessary to set memory limit.
+		log.Info("total memory is larger than 512GB, skip setting memory limit",
+			zap.Uint64("bytes", conf.GcTunerMemoryThreshold),
+			zap.String("memory", humanize.IBytes(conf.GcTunerMemoryThreshold)),
+		)
+		return
 	}
-	if conf.MaxMemoryPercentage > 0 {
-		goMemLimit := totalMemory * uint64(conf.MaxMemoryPercentage) / 100
+	if conf.GcTunerMemoryThreshold > 0 {
 		gctuner.EnableGOGCTuner.Store(true)
-		gctuner.Tuning(goMemLimit)
+		gctuner.Tuning(conf.GcTunerMemoryThreshold)
 		log.Info("enable gctuner, set memory limit",
-			zap.Uint64("bytes", goMemLimit),
-			zap.String("memory", humanize.IBytes(goMemLimit)),
+			zap.Uint64("bytes", conf.GcTunerMemoryThreshold),
+			zap.String("memory", humanize.IBytes(conf.GcTunerMemoryThreshold)),
 		)
 	}
-	return nil
 }
 
 func (s *server) startActorSystems(ctx context.Context) error {

pkg/cmd/server/server_test.go

@@ -216,7 +216,7 @@ func TestParseCfg(t *testing.T) {
 			EnableNewSink: true,
 		},
 		ClusterID:           "default",
-		MaxMemoryPercentage: config.DefaultMaxMemoryPercentage,
+		MaxMemoryPercentage: config.DisableMemoryLimit,
 	}, o.serverConfig)
 }
 
@@ -376,7 +376,7 @@ check-balance-interval = "10s"
 			EnableNewSink: true,
 		},
 		ClusterID:           "default",
-		MaxMemoryPercentage: config.DefaultMaxMemoryPercentage,
+		MaxMemoryPercentage: config.DisableMemoryLimit,
 	}, o.serverConfig)
 }
 
@@ -522,7 +522,7 @@ cert-allowed-cn = ["dd","ee"]
 			EnableNewSink: true,
 		},
 		ClusterID:           "default",
-		MaxMemoryPercentage: config.DefaultMaxMemoryPercentage,
+		MaxMemoryPercentage: config.DisableMemoryLimit,
 	}, o.serverConfig)
 }
 

pkg/config/config_test_data.go

@@ -151,7 +151,8 @@ const (
     "enable-kv-connect-backoff": false
   },
   "cluster-id": "default",
-  "max-memory-percentage": 70
+  "max-memory-percentage": 0,
+  "gc-tuner-memory-threshold": 0
 }`
 
 	testCfgTestReplicaConfigMarshal1 = `{

pkg/config/server_config.go

@@ -50,9 +50,9 @@ const (
 	// DefaultChangefeedMemoryQuota is the default memory quota for each changefeed.
 	DefaultChangefeedMemoryQuota = 1024 * 1024 * 1024 // 1GB.
 
-	// DefaultMaxMemoryPercentage is the default max memory percentage
-	// cdc server use 70% of total memory limit as soft limit by default.
-	DefaultMaxMemoryPercentage = 70
+	// DisableMemoryLimit is the default max memory percentage for TiCDC server.
+	// 0 means no memory limit.
+	DisableMemoryLimit = 0
 )
 
 var (
@@ -160,8 +160,8 @@ var defaultServerConfig = &ServerConfig{
 		EnablePullBasedSink:    true,
 		EnableKVConnectBackOff: false,
 	},
-	ClusterID:           "default",
-	MaxMemoryPercentage: DefaultMaxMemoryPercentage,
+	ClusterID:              "default",
+	GcTunerMemoryThreshold: DisableMemoryLimit,
 }
 
 // ServerConfig represents a config for server
@@ -189,7 +189,9 @@ type ServerConfig struct {
 	KVClient            *KVClientConfig `toml:"kv-client" json:"kv-client"`
 	Debug               *DebugConfig    `toml:"debug" json:"debug"`
 	ClusterID           string          `toml:"cluster-id" json:"cluster-id"`
-	MaxMemoryPercentage int             `toml:"max-memory-percentage" json:"max-memory-percentage"`
+	// Deprecated: we don't use this field anymore.
+	MaxMemoryPercentage    int    `toml:"max-memory-percentage" json:"max-memory-percentage"`
+	GcTunerMemoryThreshold uint64 `toml:"gc-tuner-memory-threshold" json:"gc-tuner-memory-threshold"`
 }
 
 // Marshal returns the json marshal format of a ServerConfig
@@ -304,11 +306,6 @@ func (c *ServerConfig) ValidateAndAdjust() error {
 	if err = c.Debug.ValidateAndAdjust(); err != nil {
 		return errors.Trace(err)
 	}
-	if c.MaxMemoryPercentage >= 100 {
-		log.Warn("server max-memory-percentage must be less than 100, set to default value")
-		c.MaxMemoryPercentage = DefaultMaxMemoryPercentage
-	}
-
 	return nil
 }