feature: add endpoint test with detailed output (#131)

cmd/mercury/mercury.go

@@ -91,6 +91,10 @@ func main() {
  os.Exit(check.GLB())
  case *param.Get().CheckBackend == true:
  os.Exit(check.Backend())
+
+ case *param.Get().CheckEndpoints == true:
+ os.Exit(check.Endpoints())
+
  }
 
  logging.Configure(config.Get().Logging.Output, config.Get().Logging.Level)

internal/check/check_endpoints.go

@@ -0,0 +1,93 @@
+package check
+
+import (
+ "github.com/schubergphilis/mercury/internal/config"
+ "github.com/schubergphilis/mercury/pkg/healthcheck"
+ "github.com/schubergphilis/mercury/pkg/logging"
+)
+
+// GLB Checks GLB status
+func Endpoints() int {
+ log := logging.For("check/endpoints")
+ log.Info("checking endpoints")
+ var expectedWorkers []*healthcheck.Worker
+
+ for poolName, pool := range config.Get().Loadbalancer.Pools {
+
+ // Create workers for pool checks
+ var poolWorkers []*healthcheck.Worker
+ for _, check := range pool.HealthChecks {
+ sourceip := pool.Listener.IP
+ if pool.Listener.SourceIP != "" {
+ sourceip = pool.Listener.SourceIP
+ }
+ worker := healthcheck.NewWorker(poolName, "", "", "", check.IP, check.Port, sourceip, check, nil)
+ poolWorkers = append(poolWorkers, worker)
+ }
+ for backendName, backend := range config.Get().Loadbalancer.Pools[poolName].Backends {
+ var backendWorkers []*healthcheck.Worker
+ // Create workers for backend checks
+ for _, check := range backend.HealthChecks {
+ if check.IP != "" {
+ sourceip := pool.Listener.IP
+ if pool.Listener.SourceIP != "" {
+ sourceip = pool.Listener.SourceIP
+ }
+ worker := healthcheck.NewWorker(poolName, backendName, "", "", check.IP, check.Port, sourceip, check, nil)
+ backendWorkers = append(backendWorkers, worker)
+ }
+ }
+
+ for _, node := range backend.Nodes {
+ var nodeWorkers []*healthcheck.Worker
+ // For each node
+ for _, check := range backend.HealthChecks {
+ if check.IP == "" {
+ sourceip := pool.Listener.IP
+ if pool.Listener.SourceIP != "" {
+ sourceip = pool.Listener.SourceIP
+ }
+
+ port := node.Port
+ if check.Port != 0 {
+ port = check.Port
+ }
+ // Create workers for node specific checks
+ worker := healthcheck.NewWorker(poolName, backendName, node.Name(), node.UUID, node.IP, port, sourceip, check, nil)
+ nodeWorkers = append(nodeWorkers, worker)
+ }
+ }
+ // Register all checks applicable to this node
+ var nodeChecks []string
+ for _, w := range nodeWorkers {
+ nodeChecks = append(nodeChecks, w.UUID())
+ }
+
+ for _, w := range backendWorkers {
+ nodeChecks = append(nodeChecks, w.UUID())
+ }
+
+ for _, w := range poolWorkers {
+ nodeChecks = append(nodeChecks, w.UUID())
+ }
+
+ // Register all checks applicable to the node UUID
+ //h.SetCheckPool(node.UUID, poolName, backendName, node.Name(), backend.HealthCheckMode, nodeChecks)
+
+ // Register worker for node checks
+ expectedWorkers = append(expectedWorkers, nodeWorkers...)
+ }
+ // Register worker for backend checks
+ expectedWorkers = append(expectedWorkers, backendWorkers...)
+ }
+ // Register worker for pool checks
+ expectedWorkers = append(expectedWorkers, poolWorkers...)
+ }
+
+ // now execute the workers
+ for _, worker := range expectedWorkers {
+ s, e, d := worker.ExecuteCheck()
+ log.Infof("worker: %+v \nstatus: %+v\n error: %+v\ndescription: %s\n\n", worker, s, e, d)
+ }
+ return OK
+}

pkg/healthcheck/healthcheck_httprequest.go

@@ -71,12 +71,12 @@ func postDataParser(t time.Time, data string) string {
 }
 
 // httpRequest does a http request check
-func httpRequest(method string, host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error) {
+func httpRequest(method string, host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error, string) {
  var err error
 
  localAddr, errl := net.ResolveIPAddr("ip", sourceIP)
  if errl != nil {
- return Offline, errl
+ return Offline, errl, fmt.Sprintf("failed to resolve %s to an ip", sourceIP)
  }
 
  localTCPAddr := net.TCPAddr{
@@ -101,7 +101,7 @@ func httpRequest(method string, host string, port int, sourceIP string, healthCh
  // Parse TLS config if provided
  tlsConfig, err := tlsconfig.LoadCertificate(healthCheck.TLSConfig)
  if err != nil {
- return Offline, fmt.Errorf("Unable to setup TLS:%s", err)
+ return Offline, fmt.Errorf("Unable to setup TLS:%s", err), fmt.Sprintf("failed to load tls configuration %v", healthCheck.TLSConfig)
  }
 
  // Overwrite default transports with our own for checking the correct node
@@ -138,7 +138,7 @@ func httpRequest(method string, host string, port int, sourceIP string, healthCh
  }
 
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("error creating request %s %s %s", method, healthCheck.HTTPRequest, postData)
  }
 
  // Process headers to add
@@ -150,36 +150,37 @@ func httpRequest(method string, host string, port int, sourceIP string, healthCh
  }
 
  req.Header.Set("User-Agent", "mercury/1.0")
+ req.Header.Set("Accept", "*/*")
  resp, err := client.Do(req)
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("error executing request %+v\n response was%+v", req, resp)
  }
 
  defer resp.Body.Close()
 
  body, err := ioutil.ReadAll(resp.Body)
  if err != nil {
- return Offline, fmt.Errorf("Error reading HTTP Body: %s", err)
+ return Offline, fmt.Errorf("Error reading HTTP Body: %s", err), fmt.Sprintf("Failed to read body, did get %+v", resp)
  }
 
  // Check health status
  if healthCheck.HTTPStatus > 0 {
  if resp.StatusCode != healthCheck.HTTPStatus {
- return Offline, fmt.Errorf("HTTP Response code incorrect (got:%d %s expected:%d)", resp.StatusCode, resp.Status, healthCheck.HTTPStatus)
+ return Offline, fmt.Errorf("HTTP Response code incorrect (got:%d %s expected:%d)", resp.StatusCode, resp.Status, healthCheck.HTTPStatus), fmt.Sprintf("Failed to get expected response, request: %+v\n return headers: %+v\n return body: %s", *req, resp, body)
  }
  }
 
  // check body
  r, err := regexp.Compile(healthCheck.HTTPReply)
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("Failed to compile regex for body check, did get headers: %+v\n body: %s", resp, body)
  }
 
  if len(healthCheck.HTTPReply) != 0 {
  if !r.MatchString(string(body)) {
- return Offline, fmt.Errorf("Reply '%s' not found in body", healthCheck.HTTPReply)
+ return Offline, fmt.Errorf("Reply '%s' not found in body", healthCheck.HTTPReply), fmt.Sprintf("Failed to find text in body, did get headers: %+v\n body: %s", resp, body)
  }
  }
  // http and body check were ok
- return Online, nil
+ return Online, nil, "all OK"
 }

pkg/healthcheck/healthcheck_ipping.go

@@ -14,7 +14,7 @@ import (
 )
 
 // tcpConnect only does a tcp connection check
-func ipPing(proto string, host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error) {
+func ipPing(proto string, host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error, string) {
  errorcount := 0
  errormsg := ""
  for i := 0; i < healthCheck.PINGpackets; i++ {
@@ -28,10 +28,10 @@ func ipPing(proto string, host string, port int, sourceIP string, healthCheck He
  }
 
  if errorcount == healthCheck.PINGpackets {
- return Offline, fmt.Errorf("%s ping lost 100%% packets: %s", proto, errormsg)
+ return Offline, fmt.Errorf("%s ping lost 100%% packets: %s", proto, errormsg), fmt.Sprintf("failed ping count")
  }
 
- return Online, nil
+ return Online, nil, "OK"
 }
 
 func pingAddr(proto string, host string, port int, sourceIP string, seq int, dataSize int, timeout time.Duration) (bool, int, error) {

pkg/healthcheck/healthcheck_ssh.go

@@ -8,7 +8,7 @@ import (
 )
 
 // tcpData does a simple tcp connect/reply check
-func sshAuth(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error) {
+func sshAuth(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error, string) {
 
  var sshConfig *ssh.ClientConfig
  if healthCheck.SSHPassword != "" {
@@ -31,12 +31,12 @@ func sshAuth(host string, port int, sourceIP string, healthCheck HealthCheck) (S
  }
  tcpAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", host, port))
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("failed to resolve to an address: %s:%d", host, port)
  }
 
  localAddr, errl := net.ResolveIPAddr("ip", sourceIP)
  if errl != nil {
- return Offline, errl
+ return Offline, errl, fmt.Sprintf("failed to resolve to an ip adress: %s", sourceIP)
  }
 
  localTCPAddr := net.TCPAddr{
@@ -46,16 +46,16 @@ func sshAuth(host string, port int, sourceIP string, healthCheck HealthCheck) (S
  // Custom dialer with
  conn, err := net.DialTCP("tcp", &localTCPAddr, tcpAddr)
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("failed to dail from source: %+v target: %+v", localTCPAddr, *tcpAddr)
  }
 
  defer conn.Close()
  _, _, _, err = ssh.NewClientConn(conn, host, sshConfig)
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("failed to initiate ssh connection on %s with %+v", host, *sshConfig)
  }
 
- return Online, nil
+ return Online, nil, "OK"
 }
 
 // publicKeyFile converts a string in to a ssh public key

pkg/healthcheck/healthcheck_tcpconnect.go

@@ -7,10 +7,10 @@ import (
 )
 
 // tcpConnect only does a tcp connection check
-func tcpConnect(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error) {
+func tcpConnect(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error, string) {
  localAddr, errl := net.ResolveIPAddr("ip", sourceIP)
  if errl != nil {
- return Offline, errl
+ return Offline, errl, fmt.Sprintf("failed to resolve to an ip adress: %s", sourceIP)
  }
 
  localTCPAddr := net.TCPAddr{
@@ -26,9 +26,9 @@ func tcpConnect(host string, port int, sourceIP string, healthCheck HealthCheck)
 
  conn, err := dialer.Dial("tcp", fmt.Sprintf("%s:%d", host, port))
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("failed to dial %s:%d", host, port)
  }
 
  conn.Close()
- return Online, nil
+ return Online, nil, "OK"
 }

pkg/healthcheck/healthcheck_tcpdata.go

@@ -9,15 +9,16 @@ import (
 )
 
 // tcpData does a simple tcp connect/reply check
-func tcpData(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error) {
+func tcpData(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error, string) {
+
  tcpAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", host, port))
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("failed to resolve to an address: %s:%d", host, port)
  }
 
  localAddr, errl := net.ResolveIPAddr("ip", sourceIP)
  if errl != nil {
- return Offline, errl
+ return Offline, errl, fmt.Sprintf("failed to resolve to an ip adress: %s", sourceIP)
  }
 
  localTCPAddr := net.TCPAddr{
@@ -27,26 +28,26 @@ func tcpData(host string, port int, sourceIP string, healthCheck HealthCheck) (S
  // Custom dialer with
  conn, err := net.DialTCP("tcp", &localTCPAddr, tcpAddr)
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("failed to dail from source: %+v target: %+v", localTCPAddr, *tcpAddr)
  }
 
  defer conn.Close()
 
  fmt.Fprintf(conn, healthCheck.TCPRequest)
  r, err := regexp.Compile(healthCheck.TCPReply)
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("regex Compile failed on %s", healthCheck.TCPReply)
  }
 
  conn.SetReadDeadline(time.Now().Add(time.Duration(healthCheck.Timeout) * time.Second))
  for {
  line, err := bufio.NewReader(conn).ReadString('\n')
  if err != nil {
- return Offline, err
+ return Offline, err, fmt.Sprintf("failed with last input %s", line)
  }
 
  if r.MatchString(line) {
- return Online, nil
+ return Online, nil, "OK"
  }
  }
 }

pkg/healthcheck/healthcheck_worker.go

@@ -119,7 +119,7 @@ func (w *Worker) Start() {
  select {
  /* new check interval has reached */
  case <-timer.C:
- result, err := w.executeCheck()
+ result, err, _ := w.ExecuteCheck()
 
  // Send update if check result or error changes
  var checkerror string
@@ -183,39 +183,40 @@ func (w *Worker) Stop() {
 }
 
 // executeCheck directs the check to the executioner and returns the result
-func (w *Worker) executeCheck() (Status, error) {
+func (w *Worker) ExecuteCheck() (Status, error, string) {
  var err error
  var result = Offline
+ var description string
 
  switch w.Check.Type {
  case "tcpconnect":
- result, err = tcpConnect(w.IP, w.Port, w.SourceIP, w.Check)
+ result, err, description = tcpConnect(w.IP, w.Port, w.SourceIP, w.Check)
 
  case "tcpdata":
- result, err = tcpData(w.IP, w.Port, w.SourceIP, w.Check)
+ result, err, description = tcpData(w.IP, w.Port, w.SourceIP, w.Check)
 
  case "ssh":
- result, err = sshAuth(w.IP, w.Port, w.SourceIP, w.Check)
+ result, err, description = sshAuth(w.IP, w.Port, w.SourceIP, w.Check)
 
  case "httpget":
- result, err = httpRequest("GET", w.IP, w.Port, w.SourceIP, w.Check)
+ result, err, description = httpRequest("GET", w.IP, w.Port, w.SourceIP, w.Check)
 
  case "httppost":
- result, err = httpRequest("POST", w.IP, w.Port, w.SourceIP, w.Check)
+ result, err, description = httpRequest("POST", w.IP, w.Port, w.SourceIP, w.Check)
 
  case "icmpping":
- result, err = ipPing("icmp", w.IP, 0, w.SourceIP, w.Check)
+ result, err, description = ipPing("icmp", w.IP, 0, w.SourceIP, w.Check)
 
  case "udpping":
- result, err = ipPing("udp", w.IP, w.Port, w.SourceIP, w.Check)
+ result, err, description = ipPing("udp", w.IP, w.Port, w.SourceIP, w.Check)
 
  case "tcpping":
- result, err = ipPing("tcp", w.IP, w.Port, w.SourceIP, w.Check)
+ result, err, description = ipPing("tcp", w.IP, w.Port, w.SourceIP, w.Check)
 
  default:
  result = Online
  }
- return result, err
+ return result, err, description
 }
 
 func (w *Worker) filterWorker() (n Worker) {