Skip to content

Commit

Permalink
feat: improve logging
Browse files Browse the repository at this point in the history
  • Loading branch information
Dmitry Selivanov committed Aug 7, 2023
1 parent 007481d commit b0602a4
Showing 1 changed file with 24 additions and 14 deletions.
38 changes: 24 additions & 14 deletions services_list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ func (l *ServicesList) Next() service.IService {
l.mu.Lock()

if len(l.healthy) == 0 {
logger.Log().Info(fmt.Sprintf("list name %s no healthy services are present during list's Next() call", l.serviceName))
return nil
}

Expand All @@ -167,20 +168,23 @@ func (l *ServicesList) Next() service.IService {
return l.healthy[idx]
}
}

logger.Log().Info(fmt.Sprintf("list name %s no healthy services are present after forloop during list's Next() call", l.serviceName))
return nil
}

// Add service to the list
func (l *ServicesList) Add(srv service.IService) {
if l.IsServiceExists(srv) {
logger.Log().Info(fmt.Sprintf("list name %s service already exists during Add, service with id %s with nodeName %s", l.serviceName, srv.ID(), srv.NodeName()))
return
}

l.mu.Lock()

if err := srv.HealthCheck(); err != nil {
l.jail[srv.ID()] = srv
logger.Log().Warn(fmt.Sprintf("can't be added to healthy pool: %s", err.Error()))
logger.Log().Warn(fmt.Sprintf("list name %s service with id %s with nodeName %s can't be added to healthy due to healthcheck error: %s", l.serviceName, srv.ID(), srv.NodeName(), err.Error()))

go l.TryUpService(srv, 0)

Expand All @@ -189,12 +193,12 @@ func (l *ServicesList) Add(srv service.IService) {
}

l.healthy = append(l.healthy, srv)
logger.Log().Info(fmt.Sprintf("%s service %s with address %s added to list", l.serviceName, srv.ID(), srv.Address()))
logger.Log().Info(fmt.Sprintf("list name %s service with id %s with nodeName %s with address %s added to list", l.serviceName, srv.ID(), srv.NodeName(), srv.Address()))
l.mu.Unlock()

if l.onSrvAddCallback != nil {
if err := l.onSrvAddCallback(srv); err != nil {
logger.Log().Warn(fmt.Sprintf("on service add callback error: %s", err.Error()))
logger.Log().Warn(fmt.Sprintf("list name %s on service add callback error: %s", l.serviceName, err.Error()))
}
}
}
Expand All @@ -221,15 +225,14 @@ func (l *ServicesList) IsServiceExists(srv service.IService) bool {
func (l *ServicesList) HealthChecks() {
for _, srv := range l.Healthy() {
if srv == nil {
logger.Log().Info(fmt.Sprintf("list name %s service is nil during hc loop, skipping the healthcheck for it", l.serviceName))
continue
}

// TODO need to implement advanced logging level

//logger.Log().Info(fmt.Sprintf("checking %s service %s...", l.serviceName, srv.ID()))

if err := srv.HealthCheck(); err != nil {
logger.Log().Warn(fmt.Errorf("healthcheck error on %s service %s: %w", l.serviceName, srv.ID(), err).Error())
logger.Log().Warn(fmt.Errorf("healthcheck error on list with name %s, service with id %s with nodeName %s: %w", l.serviceName, srv.ID(), srv.NodeName(), err).Error())

go func(service service.IService) {
l.FromHealthyToJail(service.ID())
Expand All @@ -239,8 +242,6 @@ func (l *ServicesList) HealthChecks() {

continue
}

//logger.Log().Info(fmt.Sprintf("%s service %s on %s is healthy", l.serviceName, srv.ID(), srv.Address()))
}
}

Expand All @@ -264,22 +265,22 @@ func (l *ServicesList) HealthChecksLoop() {
// TryUpService recursively try to up service
func (l *ServicesList) TryUpService(srv service.IService, try int) {
if l.TryUpTries != 0 && try >= l.TryUpTries {
logger.Log().Warn(fmt.Sprintf("maximum %d try to Up %s service %s reached.... service will remove from service list", l.TryUpTries, l.serviceName, srv.ID()))
logger.Log().Warn(fmt.Sprintf("list name %s maximum %d try to Up %s service with id %s with nodeName %s reached.... service will remove from service list", l.serviceName, l.TryUpTries, l.serviceName, srv.ID(), srv.NodeName()))
l.RemoveFromJail(srv)
return
}

logger.Log().Info(fmt.Sprintf("%d try to up %s service %s on %s", try, l.serviceName, srv.ID(), srv.Address()))
logger.Log().Info(fmt.Sprintf("list name %s %d try to up %s service with id %s with address %s with nodeName %s", l.serviceName, try, srv.ID(), srv.Address(), srv.NodeName()))

if err := srv.HealthCheck(); err != nil {
logger.Log().Warn(fmt.Errorf("service %s healthcheck error: %w", srv.ID(), err).Error())
logger.Log().Warn(fmt.Errorf("list name %s service with id %s with nodeName %s healthcheck error: %w", l.serviceName, srv.ID(), srv.NodeName(), err).Error())

Sleep(l.TryUpInterval, l.Stop)
l.TryUpService(srv, try+1)
return
}

logger.Log().Info(fmt.Sprintf("service %s is alive!", srv.ID()))
logger.Log().Info(fmt.Sprintf("list name %s service with id %s with nodeName %s is alive!", l.serviceName, srv.ID(), srv.NodeName()))

l.FromJailToHealthy(srv)
}
Expand All @@ -304,12 +305,14 @@ func (l *ServicesList) FromHealthyToJail(id string) {
}

if index == -1 {
logger.Log().Warn(fmt.Sprintf("list name %s service with id %s is not found in healthy during FromHealthyToJail", l.serviceName, id))
return
}

l.healthy = deleteFromSlice(l.healthy, index)

l.jail[srv.ID()] = srv

logger.Log().Info(fmt.Sprintf("list name %s service with id %s is moved from healthy to jail", l.serviceName, id))
}

// FromJailToHealthy move Healthy service
Expand All @@ -320,13 +323,18 @@ func (l *ServicesList) FromJailToHealthy(srv service.IService) {
l.mu.Unlock()

l.Add(srv)

logger.Log().Info(fmt.Sprintf("list name %s service with id %s with nodeName %s is moved from jail to healthy", l.serviceName, srv.ID(), srv.NodeName()))
}

func (l *ServicesList) RemoveFromHealthyByIndex(i int) {
l.mu.Lock()
defer l.mu.Unlock()

if err := l.healthy[i].Close(); err != nil {
srv := l.healthy[i]
logger.Log().Info(fmt.Sprintf("list name %s service with id %s with nodeName %s is about to be removed from healthy by index", l.serviceName, srv.ID(), srv.NodeName()))

if err := srv.Close(); err != nil {
logger.Log().Warn(fmt.Errorf("unexpected error during service Close(): %w", err).Error())
}

Expand All @@ -339,6 +347,8 @@ func (l *ServicesList) RemoveFromJail(srv service.IService) {
defer l.mu.Unlock()
l.mu.Lock()

logger.Log().Info(fmt.Sprintf("list name %s service with id %s with nodeName %s is about to be removed from jail", l.serviceName, srv.ID(), srv.NodeName()))

if err := srv.Close(); err != nil {
logger.Log().Warn(fmt.Errorf("unexpected error during service Close(): %w", err).Error())
}
Expand Down

0 comments on commit b0602a4

Please sign in to comment.