Skip to content

Commit

Permalink
move healthcheck seconds types
Browse files Browse the repository at this point in the history
Signed-off-by: Andrew Mason <andrew@planetscale.com>
  • Loading branch information
Andrew Mason committed Apr 20, 2023
1 parent 3072b54 commit df6e976
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 26 deletions.
2 changes: 1 addition & 1 deletion go/vt/vttablet/endtoend/framework/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func StartServer(connParams, connAppDebugParams mysql.ConnParams, dbName string)
_ = config.GracePeriods.ShutdownSeconds.Set("2s")
_ = config.SignalSchemaChangeReloadIntervalSeconds.Set("2100ms")
config.SignalWhenSchemaChange = true
config.Healthcheck.IntervalSeconds = 0.1
_ = config.Healthcheck.IntervalSeconds.Set("100ms")
config.Oltp.TxTimeoutSeconds = 5
config.Olap.TxTimeoutSeconds = 5
config.EnableViews = true
Expand Down
2 changes: 1 addition & 1 deletion go/vt/vttablet/tabletserver/debugenv.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func debugEnvHandler(tsv *TabletServer, w http.ResponseWriter, r *http.Request)
case "RowStreamerMaxMySQLReplLagSecs":
setInt64Val(func(val int64) { tsv.Config().RowStreamer.MaxMySQLReplLagSecs = val })
case "UnhealthyThreshold":
setDurationVal(tsv.Config().Healthcheck.UnhealthyThresholdSeconds.Set)
setDurationVal(func(d time.Duration) { _ = tsv.Config().Healthcheck.UnhealthyThresholdSeconds.Set(d.String()) })
setDurationVal(tsv.hs.SetUnhealthyThreshold)
setDurationVal(tsv.sm.SetUnhealthyThreshold)
case "ThrottleMetricThreshold":
Expand Down
55 changes: 43 additions & 12 deletions go/vt/vttablet/tabletserver/tabletenv/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,9 +174,13 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) {
_ = fs.MarkDeprecated("enable_query_plan_field_caching", "it will be removed in a future release.")
_ = fs.MarkDeprecated("enable-query-plan-field-caching", "it will be removed in a future release.")

fs.DurationVar(&healthCheckInterval, "health_check_interval", 20*time.Second, "Interval between health checks")
fs.DurationVar(&degradedThreshold, "degraded_threshold", 30*time.Second, "replication lag after which a replica is considered degraded")
fs.DurationVar(&unhealthyThreshold, "unhealthy_threshold", 2*time.Hour, "replication lag after which a replica is considered unhealthy")
currentConfig.Healthcheck.IntervalSeconds = flagutil.NewDeprecatedFloat64Seconds(defaultConfig.Healthcheck.IntervalSeconds.Name(), defaultConfig.Healthcheck.IntervalSeconds.Get())
currentConfig.Healthcheck.DegradedThresholdSeconds = flagutil.NewDeprecatedFloat64Seconds(defaultConfig.Healthcheck.DegradedThresholdSeconds.Name(), defaultConfig.Healthcheck.DegradedThresholdSeconds.Get())
currentConfig.Healthcheck.UnhealthyThresholdSeconds = flagutil.NewDeprecatedFloat64Seconds(defaultConfig.Healthcheck.UnhealthyThresholdSeconds.Name(), defaultConfig.Healthcheck.UnhealthyThresholdSeconds.Get())

fs.DurationVar(&healthCheckInterval, currentConfig.Healthcheck.IntervalSeconds.Name(), currentConfig.Healthcheck.IntervalSeconds.Get(), "Interval between health checks")
fs.DurationVar(&degradedThreshold, currentConfig.Healthcheck.DegradedThresholdSeconds.Name(), currentConfig.Healthcheck.DegradedThresholdSeconds.Get(), "replication lag after which a replica is considered degraded")
fs.DurationVar(&unhealthyThreshold, currentConfig.Healthcheck.UnhealthyThresholdSeconds.Name(), currentConfig.Healthcheck.UnhealthyThresholdSeconds.Get(), "replication lag after which a replica is considered unhealthy")
fs.DurationVar(&transitionGracePeriod, "serving_state_grace_period", 0, "how long to pause after broadcasting health to vtgate, before enforcing a new serving state")

fs.BoolVar(&enableReplicationReporter, "enable_replication_reporter", false, "Use polling to track replication lag.")
Expand Down Expand Up @@ -246,9 +250,9 @@ func Init() {
currentConfig.ReplicationTracker.Mode = Disable
}

currentConfig.Healthcheck.IntervalSeconds.Set(healthCheckInterval)
currentConfig.Healthcheck.DegradedThresholdSeconds.Set(degradedThreshold)
currentConfig.Healthcheck.UnhealthyThresholdSeconds.Set(unhealthyThreshold)
_ = currentConfig.Healthcheck.IntervalSeconds.Set(healthCheckInterval.String())
_ = currentConfig.Healthcheck.DegradedThresholdSeconds.Set(degradedThreshold.String())
_ = currentConfig.Healthcheck.UnhealthyThresholdSeconds.Set(unhealthyThreshold.String())
_ = currentConfig.GracePeriods.TransitionSeconds.Set(transitionGracePeriod.String())

switch streamlog.GetQueryLogFormat() {
Expand Down Expand Up @@ -393,9 +397,36 @@ type HotRowProtectionConfig struct {

// HealthcheckConfig contains the config for healthcheck.
type HealthcheckConfig struct {
IntervalSeconds Seconds `json:"intervalSeconds,omitempty"`
DegradedThresholdSeconds Seconds `json:"degradedThresholdSeconds,omitempty"`
UnhealthyThresholdSeconds Seconds `json:"unhealthyThresholdSeconds,omitempty"`
IntervalSeconds flagutil.DeprecatedFloat64Seconds `json:"intervalSeconds,omitempty"`
DegradedThresholdSeconds flagutil.DeprecatedFloat64Seconds `json:"degradedThresholdSeconds,omitempty"`
UnhealthyThresholdSeconds flagutil.DeprecatedFloat64Seconds `json:"unhealthyThresholdSeconds,omitempty"`
}

func (cfg *HealthcheckConfig) MarshalJSON() ([]byte, error) {
type Proxy HealthcheckConfig

tmp := struct {
Proxy
IntervalSeconds string `json:"intervalSeconds,omitempty"`
DegradedThresholdSeconds string `json:"degradedThresholdSeconds,omitempty"`
UnhealthyThresholdSeconds string `json:"unhealthyThresholdSeconds,omitempty"`
}{
Proxy: Proxy(*cfg),
}

if d := cfg.IntervalSeconds.Get(); d != 0 {
tmp.IntervalSeconds = d.String()
}

if d := cfg.DegradedThresholdSeconds.Get(); d != 0 {
tmp.DegradedThresholdSeconds = d.String()
}

if d := cfg.UnhealthyThresholdSeconds.Get(); d != 0 {
tmp.UnhealthyThresholdSeconds = d.String()
}

return json.Marshal(&tmp)
}

// GracePeriodsConfig contains various grace periods.
Expand Down Expand Up @@ -597,9 +628,9 @@ var defaultConfig = TabletConfig{
MaxRows: 10000,
},
Healthcheck: HealthcheckConfig{
IntervalSeconds: 20,
DegradedThresholdSeconds: 30,
UnhealthyThresholdSeconds: 7200,
IntervalSeconds: flagutil.NewDeprecatedFloat64Seconds("health_check_interval", 20*time.Second),
DegradedThresholdSeconds: flagutil.NewDeprecatedFloat64Seconds("degraded_threshold", 30*time.Second),
UnhealthyThresholdSeconds: flagutil.NewDeprecatedFloat64Seconds("unhealthy_threshold", 2*time.Hour),
},
GracePeriods: GracePeriodsConfig{
// TODO (ajm188) remove after these are durations. it's not necessary
Expand Down
24 changes: 12 additions & 12 deletions go/vt/vttablet/tabletserver/tabletenv/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,9 @@ consolidatorStreamQuerySize: 2097152
consolidatorStreamTotalSize: 134217728
gracePeriods: {}
healthcheck:
degradedThresholdSeconds: 30
intervalSeconds: 20
unhealthyThresholdSeconds: 7200
degradedThresholdSeconds: 30s
intervalSeconds: 20s
unhealthyThresholdSeconds: 2h0m0s
hotRowProtection:
maxConcurrency: 5
maxGlobalQueueSize: 1000
Expand Down Expand Up @@ -207,9 +207,9 @@ func TestFlags(t *testing.T) {
want.TxPool.IdleTimeoutSeconds = 1800
want.HotRowProtection.Mode = Disable
want.Consolidator = Enable
want.Healthcheck.IntervalSeconds = 20
want.Healthcheck.DegradedThresholdSeconds = 30
want.Healthcheck.UnhealthyThresholdSeconds = 7200
want.Healthcheck.IntervalSeconds.Set("20s")
want.Healthcheck.DegradedThresholdSeconds.Set("30s")
want.Healthcheck.UnhealthyThresholdSeconds.Set("2h")
want.ReplicationTracker.HeartbeatIntervalSeconds.Set("1s")
want.ReplicationTracker.Mode = Disable
assert.Equal(t, want.DB, currentConfig.DB)
Expand Down Expand Up @@ -291,21 +291,21 @@ func TestFlags(t *testing.T) {
assert.Equal(t, want, currentConfig)

healthCheckInterval = 1 * time.Second
currentConfig.Healthcheck.IntervalSeconds = 0
currentConfig.Healthcheck.IntervalSeconds.Set("0s")
Init()
want.Healthcheck.IntervalSeconds = 1
want.Healthcheck.IntervalSeconds.Set("1s")
assert.Equal(t, want, currentConfig)

degradedThreshold = 2 * time.Second
currentConfig.Healthcheck.DegradedThresholdSeconds = 0
currentConfig.Healthcheck.DegradedThresholdSeconds.Set("0s")
Init()
want.Healthcheck.DegradedThresholdSeconds = 2
want.Healthcheck.DegradedThresholdSeconds.Set("2s")
assert.Equal(t, want, currentConfig)

unhealthyThreshold = 3 * time.Second
currentConfig.Healthcheck.UnhealthyThresholdSeconds = 0
currentConfig.Healthcheck.UnhealthyThresholdSeconds.Set("0s")
Init()
want.Healthcheck.UnhealthyThresholdSeconds = 3
want.Healthcheck.UnhealthyThresholdSeconds.Set("3s")
assert.Equal(t, want, currentConfig)

transitionGracePeriod = 4 * time.Second
Expand Down

0 comments on commit df6e976

Please sign in to comment.