From 76a9fbfd5ced17e9e41202cb5c21e3a246c4022c Mon Sep 17 00:00:00 2001 From: jiangjianyuan Date: Thu, 21 Sep 2023 16:37:58 +0800 Subject: [PATCH 1/6] use TiDB gcTuner in a more reasonable way --- cdc/server/server.go | 11 +++++++++++ pkg/cmd/server/server_test.go | 6 +++--- pkg/config/config_test_data.go | 2 +- pkg/config/server_config.go | 10 +++++----- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/cdc/server/server.go b/cdc/server/server.go index 98f28e087b7..75458d31f2e 100644 --- a/cdc/server/server.go +++ b/cdc/server/server.go @@ -57,6 +57,8 @@ const ( maxHTTPConnection = 1000 // httpConnectionTimeout is used to limit a connection max alive time of http server. httpConnectionTimeout = 10 * time.Minute + // maxGcTunerMemory is used to limit the max memory usage of cdc server. if the memory is larger than it, gc tuner will be disabled + maxGcTunerMemory = 512 * 1024 * 1024 * 1024 ) // Server is the interface for the TiCDC server @@ -210,6 +212,15 @@ func (s *server) setMemoryLimit() error { if err != nil { return errors.Trace(err) } + if totalMemory > maxGcTunerMemory { + // If total memory is larger than 512GB, we will not set memory limit. + // Because the memory limit is not accurate, and it is not necessary to set memory limit. + log.Info("total memory is larger than 512GB, skip setting memory limit", + zap.Uint64("bytes", totalMemory), + zap.String("memory", humanize.IBytes(totalMemory)), + ) + return nil + } if conf.MaxMemoryPercentage > 0 { goMemLimit := totalMemory * uint64(conf.MaxMemoryPercentage) / 100 gctuner.EnableGOGCTuner.Store(true) diff --git a/pkg/cmd/server/server_test.go b/pkg/cmd/server/server_test.go index 678a868a6a4..21669fc82ea 100644 --- a/pkg/cmd/server/server_test.go +++ b/pkg/cmd/server/server_test.go @@ -200,7 +200,7 @@ func TestParseCfg(t *testing.T) { }, }, ClusterID: "default", - MaxMemoryPercentage: config.DefaultMaxMemoryPercentage, + MaxMemoryPercentage: config.DisableMemoryLimit, }, o.serverConfig) } @@ -339,7 +339,7 @@ check-balance-interval = "10s" }, }, ClusterID: "default", - MaxMemoryPercentage: config.DefaultMaxMemoryPercentage, + MaxMemoryPercentage: config.DisableMemoryLimit, }, o.serverConfig) } @@ -470,7 +470,7 @@ cert-allowed-cn = ["dd","ee"] }, }, ClusterID: "default", - MaxMemoryPercentage: config.DefaultMaxMemoryPercentage, + MaxMemoryPercentage: config.DisableMemoryLimit, }, o.serverConfig) } diff --git a/pkg/config/config_test_data.go b/pkg/config/config_test_data.go index 0a79cc042ba..b58ffc5e9a6 100644 --- a/pkg/config/config_test_data.go +++ b/pkg/config/config_test_data.go @@ -158,7 +158,7 @@ const ( } }, "cluster-id": "default", - "max-memory-percentage": 70 + "max-memory-percentage": 0 }` testCfgTestReplicaConfigMarshal1 = `{ diff --git a/pkg/config/server_config.go b/pkg/config/server_config.go index feaff34fa03..26239577384 100644 --- a/pkg/config/server_config.go +++ b/pkg/config/server_config.go @@ -45,9 +45,9 @@ const ( // DefaultChangefeedMemoryQuota is the default memory quota for each changefeed. DefaultChangefeedMemoryQuota = 1024 * 1024 * 1024 // 1GB. - // DefaultMaxMemoryPercentage is the default max memory percentage - // cdc server use 70% of total memory limit as soft limit by default. - DefaultMaxMemoryPercentage = 70 + // DisableMemoryLimit is the default max memory percentage for TiCDC server. + // 0 means no memory limit. + DisableMemoryLimit = 0 ) var ( @@ -138,7 +138,7 @@ var defaultServerConfig = &ServerConfig{ Scheduler: NewDefaultSchedulerConfig(), }, ClusterID: "default", - MaxMemoryPercentage: DefaultMaxMemoryPercentage, + MaxMemoryPercentage: DisableMemoryLimit, } // ServerConfig represents a config for server @@ -280,7 +280,7 @@ func (c *ServerConfig) ValidateAndAdjust() error { } if c.MaxMemoryPercentage >= 100 { log.Warn("server max-memory-percentage must be less than 100, set to default value") - c.MaxMemoryPercentage = DefaultMaxMemoryPercentage + c.MaxMemoryPercentage = DisableMemoryLimit } return nil From f7e067ea1b22f29ef8561fc26a12c7c402e51c43 Mon Sep 17 00:00:00 2001 From: jiangjianyuan Date: Thu, 21 Sep 2023 16:47:28 +0800 Subject: [PATCH 2/6] use TiDB gcTuner in a more reasonable way --- cdc/server/server.go | 20 +++++++------------- pkg/config/config_test_data.go | 3 ++- pkg/config/server_config.go | 13 +++++-------- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/cdc/server/server.go b/cdc/server/server.go index 75458d31f2e..1853621ab5b 100644 --- a/cdc/server/server.go +++ b/cdc/server/server.go @@ -39,7 +39,6 @@ import ( "github.com/pingcap/tiflow/pkg/p2p" "github.com/pingcap/tiflow/pkg/pdutil" "github.com/pingcap/tiflow/pkg/tcpserver" - "github.com/pingcap/tiflow/pkg/util" p2pProto "github.com/pingcap/tiflow/proto/p2p" pd "github.com/tikv/pd/client" "go.uber.org/zap" @@ -208,26 +207,21 @@ func (s *server) prepare(ctx context.Context) error { func (s *server) setMemoryLimit() error { conf := config.GetGlobalServerConfig() - totalMemory, err := util.GetMemoryLimit() - if err != nil { - return errors.Trace(err) - } - if totalMemory > maxGcTunerMemory { + if conf.MaxMemory > maxGcTunerMemory { // If total memory is larger than 512GB, we will not set memory limit. // Because the memory limit is not accurate, and it is not necessary to set memory limit. log.Info("total memory is larger than 512GB, skip setting memory limit", - zap.Uint64("bytes", totalMemory), - zap.String("memory", humanize.IBytes(totalMemory)), + zap.Uint64("bytes", conf.MaxMemory), + zap.String("memory", humanize.IBytes(conf.MaxMemory)), ) return nil } - if conf.MaxMemoryPercentage > 0 { - goMemLimit := totalMemory * uint64(conf.MaxMemoryPercentage) / 100 + if conf.MaxMemory > 0 { gctuner.EnableGOGCTuner.Store(true) - gctuner.Tuning(goMemLimit) + gctuner.Tuning(conf.MaxMemory) log.Info("enable gctuner, set memory limit", - zap.Uint64("bytes", goMemLimit), - zap.String("memory", humanize.IBytes(goMemLimit)), + zap.Uint64("bytes", conf.MaxMemory), + zap.String("memory", humanize.IBytes(conf.MaxMemory)), ) } return nil diff --git a/pkg/config/config_test_data.go b/pkg/config/config_test_data.go index b58ffc5e9a6..b13e053fdf2 100644 --- a/pkg/config/config_test_data.go +++ b/pkg/config/config_test_data.go @@ -158,7 +158,8 @@ const ( } }, "cluster-id": "default", - "max-memory-percentage": 0 + "max-memory-percentage": 0, + "max-memory": 0 }` testCfgTestReplicaConfigMarshal1 = `{ diff --git a/pkg/config/server_config.go b/pkg/config/server_config.go index 26239577384..25cc1db5542 100644 --- a/pkg/config/server_config.go +++ b/pkg/config/server_config.go @@ -137,8 +137,8 @@ var defaultServerConfig = &ServerConfig{ Scheduler: NewDefaultSchedulerConfig(), }, - ClusterID: "default", - MaxMemoryPercentage: DisableMemoryLimit, + ClusterID: "default", + MaxMemory: DisableMemoryLimit, } // ServerConfig represents a config for server @@ -167,7 +167,9 @@ type ServerConfig struct { KVClient *KVClientConfig `toml:"kv-client" json:"kv-client"` Debug *DebugConfig `toml:"debug" json:"debug"` ClusterID string `toml:"cluster-id" json:"cluster-id"` - MaxMemoryPercentage int `toml:"max-memory-percentage" json:"max-memory-percentage"` + // Deprecated: we don't use this field anymore. + MaxMemoryPercentage int `toml:"max-memory-percentage" json:"max-memory-percentage"` + MaxMemory uint64 `toml:"max-memory" json:"max-memory"` } // Marshal returns the json marshal format of a ServerConfig @@ -278,11 +280,6 @@ func (c *ServerConfig) ValidateAndAdjust() error { if err = c.Debug.ValidateAndAdjust(); err != nil { return errors.Trace(err) } - if c.MaxMemoryPercentage >= 100 { - log.Warn("server max-memory-percentage must be less than 100, set to default value") - c.MaxMemoryPercentage = DisableMemoryLimit - } - return nil } From 09ffd742f8c9787e4934d460a355d895b181191a Mon Sep 17 00:00:00 2001 From: jiangjianyuan Date: Thu, 21 Sep 2023 16:59:50 +0800 Subject: [PATCH 3/6] use TiDB gcTuner in a more reasonable way --- pkg/config/config_test_data.go | 2 +- pkg/config/server_config.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/config/config_test_data.go b/pkg/config/config_test_data.go index b13e053fdf2..47766b69301 100644 --- a/pkg/config/config_test_data.go +++ b/pkg/config/config_test_data.go @@ -159,7 +159,7 @@ const ( }, "cluster-id": "default", "max-memory-percentage": 0, - "max-memory": 0 + "gc-tuner-memory-threshold": 0 }` testCfgTestReplicaConfigMarshal1 = `{ diff --git a/pkg/config/server_config.go b/pkg/config/server_config.go index 25cc1db5542..8e9317303e9 100644 --- a/pkg/config/server_config.go +++ b/pkg/config/server_config.go @@ -137,8 +137,8 @@ var defaultServerConfig = &ServerConfig{ Scheduler: NewDefaultSchedulerConfig(), }, - ClusterID: "default", - MaxMemory: DisableMemoryLimit, + ClusterID: "default", + GcTunerMemoryThreshold: DisableMemoryLimit, } // ServerConfig represents a config for server @@ -168,8 +168,8 @@ type ServerConfig struct { Debug *DebugConfig `toml:"debug" json:"debug"` ClusterID string `toml:"cluster-id" json:"cluster-id"` // Deprecated: we don't use this field anymore. - MaxMemoryPercentage int `toml:"max-memory-percentage" json:"max-memory-percentage"` - MaxMemory uint64 `toml:"max-memory" json:"max-memory"` + MaxMemoryPercentage int `toml:"max-memory-percentage" json:"max-memory-percentage"` + GcTunerMemoryThreshold uint64 `toml:"gc-tuner-memory-threshold" json:"gc-tuner-memory-threshold"` } // Marshal returns the json marshal format of a ServerConfig From 4e2524fddb231fa890cc5442de0f8ea4d0ac5be8 Mon Sep 17 00:00:00 2001 From: jiangjianyuan Date: Thu, 21 Sep 2023 17:00:40 +0800 Subject: [PATCH 4/6] use TiDB gcTuner in a more reasonable way --- cdc/server/server.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cdc/server/server.go b/cdc/server/server.go index 1853621ab5b..de927d379b2 100644 --- a/cdc/server/server.go +++ b/cdc/server/server.go @@ -207,21 +207,21 @@ func (s *server) prepare(ctx context.Context) error { func (s *server) setMemoryLimit() error { conf := config.GetGlobalServerConfig() - if conf.MaxMemory > maxGcTunerMemory { + if conf.GcTunerMemoryThreshold > maxGcTunerMemory { // If total memory is larger than 512GB, we will not set memory limit. // Because the memory limit is not accurate, and it is not necessary to set memory limit. log.Info("total memory is larger than 512GB, skip setting memory limit", - zap.Uint64("bytes", conf.MaxMemory), - zap.String("memory", humanize.IBytes(conf.MaxMemory)), + zap.Uint64("bytes", conf.GcTunerMemoryThreshold), + zap.String("memory", humanize.IBytes(conf.GcTunerMemoryThreshold)), ) return nil } - if conf.MaxMemory > 0 { + if conf.GcTunerMemoryThreshold > 0 { gctuner.EnableGOGCTuner.Store(true) - gctuner.Tuning(conf.MaxMemory) + gctuner.Tuning(conf.GcTunerMemoryThreshold) log.Info("enable gctuner, set memory limit", - zap.Uint64("bytes", conf.MaxMemory), - zap.String("memory", humanize.IBytes(conf.MaxMemory)), + zap.Uint64("bytes", conf.GcTunerMemoryThreshold), + zap.String("memory", humanize.IBytes(conf.GcTunerMemoryThreshold)), ) } return nil From 66510f17428326415c28c578c972129e691725ba Mon Sep 17 00:00:00 2001 From: jiangjianyuan Date: Thu, 21 Sep 2023 18:59:22 +0800 Subject: [PATCH 5/6] use TiDB gcTuner in a more reasonable way --- cdc/server/server.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cdc/server/server.go b/cdc/server/server.go index de927d379b2..398d875031c 100644 --- a/cdc/server/server.go +++ b/cdc/server/server.go @@ -195,17 +195,14 @@ func (s *server) prepare(ctx context.Context) error { } s.createSortEngineFactory() - - if err := s.setMemoryLimit(); err != nil { - return errors.Trace(err) - } + s.setMemoryLimit() s.capture = capture.NewCapture(s.pdEndpoints, cdcEtcdClient, s.grpcService, s.sortEngineFactory, s.pdClient) return nil } -func (s *server) setMemoryLimit() error { +func (s *server) setMemoryLimit() { conf := config.GetGlobalServerConfig() if conf.GcTunerMemoryThreshold > maxGcTunerMemory { // If total memory is larger than 512GB, we will not set memory limit. @@ -214,7 +211,6 @@ func (s *server) setMemoryLimit() error { zap.Uint64("bytes", conf.GcTunerMemoryThreshold), zap.String("memory", humanize.IBytes(conf.GcTunerMemoryThreshold)), ) - return nil } if conf.GcTunerMemoryThreshold > 0 { gctuner.EnableGOGCTuner.Store(true) @@ -224,7 +220,6 @@ func (s *server) setMemoryLimit() error { zap.String("memory", humanize.IBytes(conf.GcTunerMemoryThreshold)), ) } - return nil } func (s *server) createSortEngineFactory() { From 5a7aa5ab80a84751f7e0df6eb2a4fd96f81210b2 Mon Sep 17 00:00:00 2001 From: jiangjianyuan Date: Fri, 22 Sep 2023 09:19:49 +0800 Subject: [PATCH 6/6] address comment --- cdc/server/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cdc/server/server.go b/cdc/server/server.go index 398d875031c..3180ee0dc10 100644 --- a/cdc/server/server.go +++ b/cdc/server/server.go @@ -211,6 +211,7 @@ func (s *server) setMemoryLimit() { zap.Uint64("bytes", conf.GcTunerMemoryThreshold), zap.String("memory", humanize.IBytes(conf.GcTunerMemoryThreshold)), ) + return } if conf.GcTunerMemoryThreshold > 0 { gctuner.EnableGOGCTuner.Store(true)