Merge pull request #29 from planetscale/ps-cherrypick-6535

Cherry-pick vitessio#6535: Add diagnostic logging to healthcheck

go/vt/discovery/healthcheck.go

@@ -278,11 +278,16 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur
 // It does not block on making connection.
 // name is an optional tag for the tablet, e.g. an alternative address.
 func (hc *HealthCheckImpl) AddTablet(tablet *topodata.Tablet) {
- log.Infof("Calling AddTablet for tablet: %v", tablet)
  // check whether we should really add this tablet
  if !hc.isIncluded(tablet) {
  return
  }
+ // check whether grpc port is present on tablet, if not return
+ if tablet.PortMap["grpc"] == 0 {
+ return
+ }
+
+ log.Infof("Adding tablet to healthcheck: %v", tablet)
  hc.mu.Lock()
  defer hc.mu.Unlock()
  if hc.healthByAlias == nil {
@@ -336,11 +341,12 @@ func (hc *HealthCheckImpl) RemoveTablet(tablet *topodata.Tablet) {
 
 // ReplaceTablet removes the old tablet and adds the new tablet.
 func (hc *HealthCheckImpl) ReplaceTablet(old, new *topodata.Tablet) {
- hc.deleteTablet(old)
+ hc.RemoveTablet(old)
  hc.AddTablet(new)
 }
 
 func (hc *HealthCheckImpl) deleteTablet(tablet *topodata.Tablet) {
+ log.Infof("Removing tablet from healthcheck: %v", tablet)
  hc.mu.Lock()
  defer hc.mu.Unlock()
 

go/vt/discovery/tablet_health_check.go

@@ -107,8 +107,9 @@ func (thc *tabletHealthCheck) setServingState(serving bool, reason string) {
  if !thc.loggedServingState || (serving != thc.Serving) {
  // Emit the log from a separate goroutine to avoid holding
  // the th lock while logging is happening
- go log.Infof("HealthCheckUpdate(Serving State): tablet: %v serving => %v for %v/%v (%v) reason: %s",
+ log.Infof("HealthCheckUpdate(Serving State): tablet: %v serving %v => %v for %v/%v (%v) reason: %s",
  topotools.TabletIdent(thc.Tablet),
+ thc.Serving,
  serving,
  thc.Tablet.GetKeyspace(),
  thc.Tablet.GetShard(),

go/vt/vtgate/vschema_manager.go

@@ -71,6 +71,7 @@ func (vm *VSchemaManager) watchSrvVSchema(ctx context.Context, cell string) {
  // we don't know the real value. In this case, we want
  // to use the previous value if it was set, or an
  // empty vschema if it wasn't.
+ log.Infof("Received vschema update")
  switch {
  case err == nil:
  // Good case, we can try to save that value.
@@ -79,6 +80,7 @@ func (vm *VSchemaManager) watchSrvVSchema(ctx context.Context, cell string) {
  // Otherwise, keep what we already had before.
  v = nil
  default:
+ log.Errorf("SrvVschema watch error: %v", err)
  // Watch error, increment our counters.
  if vschemaCounters != nil {
  vschemaCounters.Add("WatchError", 1)