Merge pull request #6320 from planetscale/ds-6.0-backport-6250

backport #6250 to 6.0 release branch

go.mod

@@ -47,6 +47,7 @@ require (
 	github.com/krishicks/yaml-patch v0.0.10
 	github.com/magiconair/properties v1.8.1
 	github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1
+	github.com/mitchellh/go-ps v1.0.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
 	github.com/onsi/ginkgo v1.10.3 // indirect

go.sum

@@ -378,6 +378,8 @@ github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1/go.mod h1:vuvdOZLJu
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/mitchellh/go-testing-interface v1.14.0 h1:/x0XQ6h+3U3nAyk1yx+bHPURrKa9sVVvYbuqZ7pIAtI=
 github.com/mitchellh/go-testing-interface v1.14.0/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=

go/vt/vttablet/endtoend/config_test.go

@@ -60,7 +60,7 @@ func TestConfigVars(t *testing.T) {
 		val: currentConfig.OltpReadPool.Size,
 	}, {
 		tag: "ConnPoolIdleTimeout",
-		val: currentConfig.OltpReadPool.IdleTimeoutSeconds * 1e9,
+		val: int(currentConfig.OltpReadPool.IdleTimeoutSeconds * 1e9),
 	}, {
 		tag: "ConnPoolMaxCap",
 		val: currentConfig.OltpReadPool.Size,
@@ -90,7 +90,7 @@ func TestConfigVars(t *testing.T) {
 		val: currentConfig.OlapReadPool.Size,
 	}, {
 		tag: "StreamConnPoolIdleTimeout",
-		val: currentConfig.OlapReadPool.IdleTimeoutSeconds * 1e9,
+		val: int(currentConfig.OlapReadPool.IdleTimeoutSeconds * 1e9),
 	}, {
 		tag: "StreamConnPoolMaxCap",
 		val: currentConfig.OlapReadPool.Size,
@@ -102,7 +102,7 @@ func TestConfigVars(t *testing.T) {
 		val: currentConfig.TxPool.Size,
 	}, {
 		tag: "TransactionPoolIdleTimeout",
-		val: currentConfig.TxPool.IdleTimeoutSeconds * 1e9,
+		val: int(currentConfig.TxPool.IdleTimeoutSeconds * 1e9),
 	}, {
 		tag: "TransactionPoolMaxCap",
 		val: currentConfig.TxPool.Size,

go/vt/vttablet/heartbeat/reader.go

@@ -70,11 +70,11 @@ type Reader struct {
 // NewReader returns a new heartbeat reader.
 func NewReader(env tabletenv.Env) *Reader {
 	config := env.Config()
-	if config.HeartbeatIntervalMilliseconds == 0 {
+	if config.HeartbeatIntervalSeconds == 0 {
 		return &Reader{}
 	}
 
-	heartbeatInterval := time.Duration(config.HeartbeatIntervalMilliseconds) * time.Millisecond
+	heartbeatInterval := time.Duration(config.HeartbeatIntervalSeconds * 1e9)
 	return &Reader{
 		env:      env,
 		enabled:  true,

go/vt/vttablet/heartbeat/reader_test.go

@@ -100,7 +100,7 @@ func TestReaderReadHeartbeatError(t *testing.T) {
 
 func newReader(db *fakesqldb.DB, nowFunc func() time.Time) *Reader {
 	config := tabletenv.NewDefaultConfig()
-	config.HeartbeatIntervalMilliseconds = 1000
+	config.HeartbeatIntervalSeconds = 1
 	params, _ := db.ConnParams().MysqlParams()
 	cp := *params
 	dbc := dbconfigs.NewTestDBConfigs(cp, cp, "")

go/vt/vttablet/heartbeat/writer.go

@@ -71,10 +71,10 @@ type Writer struct {
 // NewWriter creates a new Writer.
 func NewWriter(env tabletenv.Env, alias topodatapb.TabletAlias) *Writer {
 	config := env.Config()
-	if config.HeartbeatIntervalMilliseconds == 0 {
+	if config.HeartbeatIntervalSeconds == 0 {
 		return &Writer{}
 	}
-	heartbeatInterval := time.Duration(config.HeartbeatIntervalMilliseconds) * time.Millisecond
+	heartbeatInterval := time.Duration(config.HeartbeatIntervalSeconds * 1e9)
 	return &Writer{
 		env:         env,
 		enabled:     true,

go/vt/vttablet/heartbeat/writer_test.go

@@ -104,7 +104,7 @@ func TestWriteHeartbeatError(t *testing.T) {
 
 func newTestWriter(db *fakesqldb.DB, nowFunc func() time.Time) *Writer {
 	config := tabletenv.NewDefaultConfig()
-	config.HeartbeatIntervalMilliseconds = 1000
+	config.HeartbeatIntervalSeconds = 1
 
 	params, _ := db.ConnParams().MysqlParams()
 	cp := *params

go/vt/vttablet/tabletmanager/heartbeat_reporter.go

@@ -34,7 +34,7 @@ type Reporter struct {
 // RegisterReporter registers the heartbeat reader as a healthcheck reporter so that its
 // measurements will be picked up in healthchecks.
 func registerHeartbeatReporter(controller tabletserver.Controller) {
-	if tabletenv.NewCurrentConfig().HeartbeatIntervalMilliseconds == 0 {
+	if tabletenv.NewCurrentConfig().HeartbeatIntervalSeconds == 0 {
 		return
 	}
 

go/vt/vttablet/tabletserver/query_engine_test.go

@@ -275,9 +275,9 @@ func TestStatsURL(t *testing.T) {
 func newTestQueryEngine(queryCacheSize int, idleTimeout time.Duration, strict bool, dbcfgs *dbconfigs.DBConfigs) *QueryEngine {
 	config := tabletenv.NewDefaultConfig()
 	config.QueryCacheSize = queryCacheSize
-	config.OltpReadPool.IdleTimeoutSeconds = int(idleTimeout / 1e9)
-	config.OlapReadPool.IdleTimeoutSeconds = int(idleTimeout / 1e9)
-	config.TxPool.IdleTimeoutSeconds = int(idleTimeout / 1e9)
+	config.OltpReadPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9)
+	config.OlapReadPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9)
+	config.TxPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9)
 	env := tabletenv.NewTestEnv(config, dbcfgs, "TabletServerTest")
 	se := schema.NewEngine(env)
 	qe := NewQueryEngine(env, se)

go/vt/vttablet/tabletserver/schema/engine_test.go

@@ -301,10 +301,10 @@ func TestStatsURL(t *testing.T) {
 func newEngine(queryCacheSize int, reloadTime time.Duration, idleTimeout time.Duration, strict bool, db *fakesqldb.DB) *Engine {
 	config := tabletenv.NewDefaultConfig()
 	config.QueryCacheSize = queryCacheSize
-	config.SchemaReloadIntervalSeconds = int(reloadTime) / 1e9
-	config.OltpReadPool.IdleTimeoutSeconds = int(idleTimeout / 1e9)
-	config.OlapReadPool.IdleTimeoutSeconds = int(idleTimeout / 1e9)
-	config.TxPool.IdleTimeoutSeconds = int(idleTimeout / 1e9)
+	config.SchemaReloadIntervalSeconds = float64(reloadTime) / 1e9
+	config.OltpReadPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9)
+	config.OlapReadPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9)
+	config.TxPool.IdleTimeoutSeconds = float64(idleTimeout / 1e9)
 	se := NewEngine(tabletenv.NewTestEnv(config, nil, "SchemaTest"))
 	se.InitDBConfig(newDBConfigs(db).DbaWithDB())
 	return se

go/vt/vttablet/tabletserver/tabletenv/config.go

@@ -85,8 +85,8 @@ func init() {
 	flag.IntVar(&currentConfig.TxPool.PrefillParallelism, "queryserver-config-transaction-prefill-parallelism", defaultConfig.TxPool.PrefillParallelism, "query server transaction prefill parallelism, a non-zero value will prefill the pool using the specified parallism.")
 	flag.IntVar(&currentConfig.MessagePostponeParallelism, "queryserver-config-message-postpone-cap", defaultConfig.MessagePostponeParallelism, "query server message postpone cap is the maximum number of messages that can be postponed at any given time. Set this number to substantially lower than transaction cap, so that the transaction pool isn't exhausted by the message subsystem.")
 	flag.IntVar(&deprecatedFoundRowsPoolSize, "client-found-rows-pool-size", 0, "DEPRECATED: queryserver-config-transaction-cap will be used instead.")
-	flag.IntVar(&currentConfig.Oltp.TxTimeoutSeconds, "queryserver-config-transaction-timeout", defaultConfig.Oltp.TxTimeoutSeconds, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value")
-	flag.IntVar(&currentConfig.ShutdownGracePeriodSeconds, "transaction_shutdown_grace_period", defaultConfig.ShutdownGracePeriodSeconds, "how long to wait (in seconds) for transactions to complete during graceful shutdown.")
+	flag.Float64Var(&currentConfig.Oltp.TxTimeoutSeconds, "queryserver-config-transaction-timeout", defaultConfig.Oltp.TxTimeoutSeconds, "query server transaction timeout (in seconds), a transaction will be killed if it takes longer than this value")
+	flag.Float64Var(&currentConfig.ShutdownGracePeriodSeconds, "transaction_shutdown_grace_period", defaultConfig.ShutdownGracePeriodSeconds, "how long to wait (in seconds) for transactions to complete during graceful shutdown.")
 	flag.IntVar(&currentConfig.Oltp.MaxRows, "queryserver-config-max-result-size", defaultConfig.Oltp.MaxRows, "query server max result size, maximum number of rows allowed to return from vttablet for non-streaming queries.")
 	flag.IntVar(&currentConfig.Oltp.WarnRows, "queryserver-config-warn-result-size", defaultConfig.Oltp.WarnRows, "query server result size warning threshold, warn if number of rows returned from vttablet for non-streaming queries exceeds this")
 	flag.IntVar(&deprecatedMaxDMLRows, "queryserver-config-max-dml-rows", 0, "query server max dml rows per statement, maximum number of rows allowed to return at a time for an update or delete with either 1) an equality where clauses on primary keys, or 2) a subselect statement. For update and delete statements in above two categories, vttablet will split the original query into multiple small queries based on this configuration value. ")
@@ -95,11 +95,12 @@ func init() {
 
 	flag.IntVar(&currentConfig.StreamBufferSize, "queryserver-config-stream-buffer-size", defaultConfig.StreamBufferSize, "query server stream buffer size, the maximum number of bytes sent from vttablet for each stream call. It's recommended to keep this value in sync with vtgate's stream_buffer_size.")
 	flag.IntVar(&currentConfig.QueryCacheSize, "queryserver-config-query-cache-size", defaultConfig.QueryCacheSize, "query server query cache size, maximum number of queries to be cached. vttablet analyzes every incoming query and generate a query plan, these plans are being cached in a lru cache. This config controls the capacity of the lru cache.")
-	flag.IntVar(&currentConfig.SchemaReloadIntervalSeconds, "queryserver-config-schema-reload-time", defaultConfig.SchemaReloadIntervalSeconds, "query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.")
-	flag.IntVar(&currentConfig.Oltp.QueryTimeoutSeconds, "queryserver-config-query-timeout", defaultConfig.Oltp.QueryTimeoutSeconds, "query server query timeout (in seconds), this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed.")
-	flag.IntVar(&currentConfig.OltpReadPool.TimeoutSeconds, "queryserver-config-query-pool-timeout", defaultConfig.OltpReadPool.TimeoutSeconds, "query server query pool timeout (in seconds), it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead.")
-	flag.IntVar(&currentConfig.TxPool.TimeoutSeconds, "queryserver-config-txpool-timeout", defaultConfig.TxPool.TimeoutSeconds, "query server transaction pool timeout, it is how long vttablet waits if tx pool is full")
-	flag.IntVar(&currentConfig.OltpReadPool.IdleTimeoutSeconds, "queryserver-config-idle-timeout", defaultConfig.OltpReadPool.IdleTimeoutSeconds, "query server idle timeout (in seconds), vttablet manages various mysql connection pools. This config means if a connection has not been used in given idle timeout, this connection will be removed from pool. This effectively manages number of connection objects and optimize the pool performance.")
+	flag.Float64Var(&currentConfig.SchemaReloadIntervalSeconds, "queryserver-config-schema-reload-time", defaultConfig.SchemaReloadIntervalSeconds, "query server schema reload time, how often vttablet reloads schemas from underlying MySQL instance in seconds. vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.")
+	flag.Float64Var(&currentConfig.Oltp.QueryTimeoutSeconds, "queryserver-config-query-timeout", defaultConfig.Oltp.QueryTimeoutSeconds, "query server query timeout (in seconds), this is the query timeout in vttablet side. If a query takes more than this timeout, it will be killed.")
+	flag.Float64Var(&currentConfig.OltpReadPool.TimeoutSeconds, "queryserver-config-query-pool-timeout", defaultConfig.OltpReadPool.TimeoutSeconds, "query server query pool timeout (in seconds), it is how long vttablet waits for a connection from the query pool. If set to 0 (default) then the overall query timeout is used instead.")
+	flag.Float64Var(&currentConfig.OlapReadPool.TimeoutSeconds, "queryserver-config-stream-pool-timeout", defaultConfig.OlapReadPool.TimeoutSeconds, "query server stream pool timeout (in seconds), it is how long vttablet waits for a connection from the stream pool. If set to 0 (default) then there is no timeout.")
+	flag.Float64Var(&currentConfig.TxPool.TimeoutSeconds, "queryserver-config-txpool-timeout", defaultConfig.TxPool.TimeoutSeconds, "query server transaction pool timeout, it is how long vttablet waits if tx pool is full")
+	flag.Float64Var(&currentConfig.OltpReadPool.IdleTimeoutSeconds, "queryserver-config-idle-timeout", defaultConfig.OltpReadPool.IdleTimeoutSeconds, "query server idle timeout (in seconds), vttablet manages various mysql connection pools. This config means if a connection has not been used in given idle timeout, this connection will be removed from pool. This effectively manages number of connection objects and optimize the pool performance.")
 	flag.IntVar(&currentConfig.OltpReadPool.MaxWaiters, "queryserver-config-query-pool-waiter-cap", defaultConfig.OltpReadPool.MaxWaiters, "query server query pool waiter limit, this is the maximum number of queries that can be queued waiting to get a connection")
 	flag.IntVar(&currentConfig.TxPool.MaxWaiters, "queryserver-config-txpool-waiter-cap", defaultConfig.TxPool.MaxWaiters, "query server transaction pool waiter limit, this is the maximum number of transactions that can be queued waiting to get a connection")
 	// tableacl related configurations.
@@ -167,7 +168,7 @@ func Init() {
 	}
 
 	if enableHeartbeat {
-		currentConfig.HeartbeatIntervalMilliseconds = int(heartbeatInterval / time.Millisecond)
+		currentConfig.HeartbeatIntervalSeconds = float64(heartbeatInterval) / float64(time.Millisecond)
 	}
 
 	switch *streamlog.QueryLogFormat {
@@ -190,22 +191,24 @@ func Init() {
 type TabletConfig struct {
 	DB *dbconfigs.DBConfigs `json:"db,omitempty"`
 
-	OltpReadPool                  ConnPoolConfig         `json:"oltpReadPool,omitempty"`
-	OlapReadPool                  ConnPoolConfig         `json:"olapReadPool,omitempty"`
-	TxPool                        ConnPoolConfig         `json:"txPool,omitempty"`
-	Oltp                          OltpConfig             `json:"oltp,omitempty"`
-	HotRowProtection              HotRowProtectionConfig `json:"hotRowProtection,omitempty"`
-	Consolidator                  string                 `json:"consolidator,omitempty"`
-	HeartbeatIntervalMilliseconds int                    `json:"heartbeatIntervalMilliseconds,omitempty"`
-	ShutdownGracePeriodSeconds    int                    `json:"shutdownGracePeriodSeconds,omitempty"`
-	PassthroughDML                bool                   `json:"passthroughDML,omitempty"`
-	StreamBufferSize              int                    `json:"streamBufferSize,omitempty"`
-	QueryCacheSize                int                    `json:"queryCacheSize,omitempty"`
-	SchemaReloadIntervalSeconds   int                    `json:"schemaReloadIntervalSeconds,omitempty"`
-	WatchReplication              bool                   `json:"watchReplication,omitempty"`
-	TerseErrors                   bool                   `json:"terseErrors,omitempty"`
-	MessagePostponeParallelism    int                    `json:"messagePostponeParallelism,omitempty"`
-	CacheResultFields             bool                   `json:"cacheResultFields,omitempty"`
+	OltpReadPool ConnPoolConfig `json:"oltpReadPool,omitempty"`
+	OlapReadPool ConnPoolConfig `json:"olapReadPool,omitempty"`
+	TxPool       ConnPoolConfig `json:"txPool,omitempty"`
+
+	Oltp             OltpConfig             `json:"oltp,omitempty"`
+	HotRowProtection HotRowProtectionConfig `json:"hotRowProtection,omitempty"`
+
+	Consolidator                string  `json:"consolidator,omitempty"`
+	HeartbeatIntervalSeconds    float64 `json:"heartbeatIntervalSeconds,omitempty"`
+	ShutdownGracePeriodSeconds  float64 `json:"shutdownGracePeriodSeconds,omitempty"`
+	PassthroughDML              bool    `json:"passthroughDML,omitempty"`
+	StreamBufferSize            int     `json:"streamBufferSize,omitempty"`
+	QueryCacheSize              int     `json:"queryCacheSize,omitempty"`
+	SchemaReloadIntervalSeconds float64 `json:"schemaReloadIntervalSeconds,omitempty"`
+	WatchReplication            bool    `json:"watchReplication,omitempty"`
+	TerseErrors                 bool    `json:"terseErrors,omitempty"`
+	MessagePostponeParallelism  int     `json:"messagePostponeParallelism,omitempty"`
+	CacheResultFields           bool    `json:"cacheResultFields,omitempty"`
 
 	StrictTableACL          bool    `json:"-"`
 	EnableTableACLDryRun    bool    `json:"-"`
@@ -225,19 +228,19 @@ type TabletConfig struct {
 
 // ConnPoolConfig contains the config for a conn pool.
 type ConnPoolConfig struct {
-	Size               int `json:"size,omitempty"`
-	TimeoutSeconds     int `json:"timeoutSeconds,omitempty"`
-	IdleTimeoutSeconds int `json:"idleTimeoutSeconds,omitempty"`
-	PrefillParallelism int `json:"prefillParallelism,omitempty"`
-	MaxWaiters         int `json:"maxWaiters,omitempty"`
+	Size               int     `json:"size,omitempty"`
+	TimeoutSeconds     float64 `json:"timeoutSeconds,omitempty"`
+	IdleTimeoutSeconds float64 `json:"idleTimeoutSeconds,omitempty"`
+	PrefillParallelism int     `json:"prefillParallelism,omitempty"`
+	MaxWaiters         int     `json:"maxWaiters,omitempty"`
 }
 
 // OltpConfig contains the config for oltp settings.
 type OltpConfig struct {
-	QueryTimeoutSeconds int `json:"queryTimeoutSeconds,omitempty"`
-	TxTimeoutSeconds    int `json:"txTimeoutSeconds,omitempty"`
-	MaxRows             int `json:"maxRpws,omitempty"`
-	WarnRows            int `json:"warnRows,omitempty"`
+	QueryTimeoutSeconds float64 `json:"queryTimeoutSeconds,omitempty"`
+	TxTimeoutSeconds    float64 `json:"txTimeoutSeconds,omitempty"`
+	MaxRows             int     `json:"maxRpws,omitempty"`
+	WarnRows            int     `json:"warnRows,omitempty"`
 }
 
 // HotRowProtectionConfig contains the config for hot row protection.