From b4866778525b775957168a8f181db1b6efea811d Mon Sep 17 00:00:00 2001 From: Michael Demmer Date: Wed, 17 Jan 2018 06:24:05 -0800 Subject: [PATCH 1/3] refactor mysql protocol shutdown to enable testing --- go/vt/vtgate/plugin_mysql_server.go | 35 +++++++++++++++-------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/go/vt/vtgate/plugin_mysql_server.go b/go/vt/vtgate/plugin_mysql_server.go index ea75a7a0579..be8df0b0230 100644 --- a/go/vt/vtgate/plugin_mysql_server.go +++ b/go/vt/vtgate/plugin_mysql_server.go @@ -230,26 +230,27 @@ func newMysqlUnixSocket(address string, authServer mysql.AuthServer, handler mys } } -func init() { - servenv.OnRun(initMySQLProtocol) +func shutdownMysqlProtocolAndDrain() { + if mysqlListener != nil { + mysqlListener.Close() + mysqlListener = nil + } + if mysqlUnixListener != nil { + mysqlUnixListener.Close() + mysqlUnixListener = nil + } - servenv.OnTermSync(func() { - if mysqlListener != nil { - mysqlListener.Close() - mysqlListener = nil - } - if mysqlUnixListener != nil { - mysqlUnixListener.Close() - mysqlUnixListener = nil + if atomic.LoadInt32(&busyConnections) > 0 { + log.Infof("Waiting for all client connections to be idle...") + for atomic.LoadInt32(&busyConnections) > 0 { + time.Sleep(1 * time.Millisecond) } + } +} - if atomic.LoadInt32(&busyConnections) > 0 { - log.Infof("Waiting for all client connections to be idle...") - for atomic.LoadInt32(&busyConnections) > 0 { - time.Sleep(1 * time.Millisecond) - } - } - }) +func init() { + servenv.OnRun(initMySQLProtocol) + servenv.OnTermSync(shutdownMysqlProtocolAndDrain) } var pluginInitializers []func() From b9d32c124365d2a287b099653c09a5e4b1b01524 Mon Sep 17 00:00:00 2001 From: Michael Demmer Date: Wed, 17 Jan 2018 06:50:12 -0800 Subject: [PATCH 2/3] add more logging and a max wait time to vtgate shutdown drain Wait no more than 30 seconds for active vtgate mysql connections to drain, and emit a log every 2 seconds while waiting with the count of connections that we're waiting for. --- go/vt/vtgate/plugin_mysql_server.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/go/vt/vtgate/plugin_mysql_server.go b/go/vt/vtgate/plugin_mysql_server.go index be8df0b0230..81b581873cd 100644 --- a/go/vt/vtgate/plugin_mysql_server.go +++ b/go/vt/vtgate/plugin_mysql_server.go @@ -241,10 +241,25 @@ func shutdownMysqlProtocolAndDrain() { } if atomic.LoadInt32(&busyConnections) > 0 { - log.Infof("Waiting for all client connections to be idle...") - for atomic.LoadInt32(&busyConnections) > 0 { + log.Infof("Waiting for all client connections to be idle (%d active)...", atomic.LoadInt32(&busyConnections)) + start := time.Now() + reported := start + for time.Since(start) < 30*time.Second { + if atomic.LoadInt32(&busyConnections) == 0 { + break + } + + if time.Since(reported) > 2*time.Second { + log.Infof("Still waiting for client connections to be idle (%d active)...", atomic.LoadInt32(&busyConnections)) + reported = time.Now() + } + time.Sleep(1 * time.Millisecond) } + + if atomic.LoadInt32(&busyConnections) > 0 { + log.Infof("Client connections still not idle, exiting anyway (%d active)...", atomic.LoadInt32(&busyConnections)) + } } } From dd2be93cf458a02ff324260f8706d904a536d643 Mon Sep 17 00:00:00 2001 From: Michael Demmer Date: Tue, 23 Jan 2018 16:00:06 -0800 Subject: [PATCH 3/3] remove the explicit shutdown timer since the OnTermSync has its own --- go/vt/vtgate/plugin_mysql_server.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/go/vt/vtgate/plugin_mysql_server.go b/go/vt/vtgate/plugin_mysql_server.go index 81b581873cd..b0627ed9bae 100644 --- a/go/vt/vtgate/plugin_mysql_server.go +++ b/go/vt/vtgate/plugin_mysql_server.go @@ -244,11 +244,7 @@ func shutdownMysqlProtocolAndDrain() { log.Infof("Waiting for all client connections to be idle (%d active)...", atomic.LoadInt32(&busyConnections)) start := time.Now() reported := start - for time.Since(start) < 30*time.Second { - if atomic.LoadInt32(&busyConnections) == 0 { - break - } - + for atomic.LoadInt32(&busyConnections) != 0 { if time.Since(reported) > 2*time.Second { log.Infof("Still waiting for client connections to be idle (%d active)...", atomic.LoadInt32(&busyConnections)) reported = time.Now() @@ -256,10 +252,6 @@ func shutdownMysqlProtocolAndDrain() { time.Sleep(1 * time.Millisecond) } - - if atomic.LoadInt32(&busyConnections) > 0 { - log.Infof("Client connections still not idle, exiting anyway (%d active)...", atomic.LoadInt32(&busyConnections)) - } } }