diff --git a/cmd/mercury/mercury.go b/cmd/mercury/mercury.go index ba21fab..a91eb10 100644 --- a/cmd/mercury/mercury.go +++ b/cmd/mercury/mercury.go @@ -91,6 +91,10 @@ func main() { os.Exit(check.GLB()) case *param.Get().CheckBackend == true: os.Exit(check.Backend()) + + case *param.Get().CheckEndpoints == true: + os.Exit(check.Endpoints()) + } logging.Configure(config.Get().Logging.Output, config.Get().Logging.Level) diff --git a/internal/check/check_endpoints.go b/internal/check/check_endpoints.go new file mode 100644 index 0000000..e3ab690 --- /dev/null +++ b/internal/check/check_endpoints.go @@ -0,0 +1,93 @@ +package check + +import ( + "github.com/schubergphilis/mercury/internal/config" + "github.com/schubergphilis/mercury/pkg/healthcheck" + "github.com/schubergphilis/mercury/pkg/logging" +) + +// GLB Checks GLB status +func Endpoints() int { + log := logging.For("check/endpoints") + log.Info("checking endpoints") + var expectedWorkers []*healthcheck.Worker + + for poolName, pool := range config.Get().Loadbalancer.Pools { + + // Create workers for pool checks + var poolWorkers []*healthcheck.Worker + for _, check := range pool.HealthChecks { + sourceip := pool.Listener.IP + if pool.Listener.SourceIP != "" { + sourceip = pool.Listener.SourceIP + } + worker := healthcheck.NewWorker(poolName, "", "", "", check.IP, check.Port, sourceip, check, nil) + poolWorkers = append(poolWorkers, worker) + } + for backendName, backend := range config.Get().Loadbalancer.Pools[poolName].Backends { + var backendWorkers []*healthcheck.Worker + // Create workers for backend checks + for _, check := range backend.HealthChecks { + if check.IP != "" { + sourceip := pool.Listener.IP + if pool.Listener.SourceIP != "" { + sourceip = pool.Listener.SourceIP + } + worker := healthcheck.NewWorker(poolName, backendName, "", "", check.IP, check.Port, sourceip, check, nil) + backendWorkers = append(backendWorkers, worker) + } + } + + for _, node := range backend.Nodes { + var nodeWorkers []*healthcheck.Worker + // For each node + for _, check := range backend.HealthChecks { + if check.IP == "" { + sourceip := pool.Listener.IP + if pool.Listener.SourceIP != "" { + sourceip = pool.Listener.SourceIP + } + + port := node.Port + if check.Port != 0 { + port = check.Port + } + // Create workers for node specific checks + worker := healthcheck.NewWorker(poolName, backendName, node.Name(), node.UUID, node.IP, port, sourceip, check, nil) + nodeWorkers = append(nodeWorkers, worker) + } + } + // Register all checks applicable to this node + var nodeChecks []string + for _, w := range nodeWorkers { + nodeChecks = append(nodeChecks, w.UUID()) + } + + for _, w := range backendWorkers { + nodeChecks = append(nodeChecks, w.UUID()) + } + + for _, w := range poolWorkers { + nodeChecks = append(nodeChecks, w.UUID()) + } + + // Register all checks applicable to the node UUID + //h.SetCheckPool(node.UUID, poolName, backendName, node.Name(), backend.HealthCheckMode, nodeChecks) + + // Register worker for node checks + expectedWorkers = append(expectedWorkers, nodeWorkers...) + } + // Register worker for backend checks + expectedWorkers = append(expectedWorkers, backendWorkers...) + } + // Register worker for pool checks + expectedWorkers = append(expectedWorkers, poolWorkers...) + } + + // now execute the workers + for _, worker := range expectedWorkers { + s, e, d := worker.ExecuteCheck() + log.Infof("worker: %+v \nstatus: %+v\n error: %+v\ndescription: %s\n\n", worker, s, e, d) + } + return OK +} diff --git a/pkg/healthcheck/healthcheck_httprequest.go b/pkg/healthcheck/healthcheck_httprequest.go index 9a2041c..dba10a6 100644 --- a/pkg/healthcheck/healthcheck_httprequest.go +++ b/pkg/healthcheck/healthcheck_httprequest.go @@ -71,12 +71,12 @@ func postDataParser(t time.Time, data string) string { } // httpRequest does a http request check -func httpRequest(method string, host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error) { +func httpRequest(method string, host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error, string) { var err error localAddr, errl := net.ResolveIPAddr("ip", sourceIP) if errl != nil { - return Offline, errl + return Offline, errl, fmt.Sprintf("failed to resolve %s to an ip", sourceIP) } localTCPAddr := net.TCPAddr{ @@ -101,7 +101,7 @@ func httpRequest(method string, host string, port int, sourceIP string, healthCh // Parse TLS config if provided tlsConfig, err := tlsconfig.LoadCertificate(healthCheck.TLSConfig) if err != nil { - return Offline, fmt.Errorf("Unable to setup TLS:%s", err) + return Offline, fmt.Errorf("Unable to setup TLS:%s", err), fmt.Sprintf("failed to load tls configuration %v", healthCheck.TLSConfig) } // Overwrite default transports with our own for checking the correct node @@ -138,7 +138,7 @@ func httpRequest(method string, host string, port int, sourceIP string, healthCh } if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("error creating request %s %s %s", method, healthCheck.HTTPRequest, postData) } // Process headers to add @@ -150,36 +150,37 @@ func httpRequest(method string, host string, port int, sourceIP string, healthCh } req.Header.Set("User-Agent", "mercury/1.0") + req.Header.Set("Accept", "*/*") resp, err := client.Do(req) if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("error executing request %+v\n response was%+v", req, resp) } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - return Offline, fmt.Errorf("Error reading HTTP Body: %s", err) + return Offline, fmt.Errorf("Error reading HTTP Body: %s", err), fmt.Sprintf("Failed to read body, did get %+v", resp) } // Check health status if healthCheck.HTTPStatus > 0 { if resp.StatusCode != healthCheck.HTTPStatus { - return Offline, fmt.Errorf("HTTP Response code incorrect (got:%d %s expected:%d)", resp.StatusCode, resp.Status, healthCheck.HTTPStatus) + return Offline, fmt.Errorf("HTTP Response code incorrect (got:%d %s expected:%d)", resp.StatusCode, resp.Status, healthCheck.HTTPStatus), fmt.Sprintf("Failed to get expected response, request: %+v\n return headers: %+v\n return body: %s", *req, resp, body) } } // check body r, err := regexp.Compile(healthCheck.HTTPReply) if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("Failed to compile regex for body check, did get headers: %+v\n body: %s", resp, body) } if len(healthCheck.HTTPReply) != 0 { if !r.MatchString(string(body)) { - return Offline, fmt.Errorf("Reply '%s' not found in body", healthCheck.HTTPReply) + return Offline, fmt.Errorf("Reply '%s' not found in body", healthCheck.HTTPReply), fmt.Sprintf("Failed to find text in body, did get headers: %+v\n body: %s", resp, body) } } // http and body check were ok - return Online, nil + return Online, nil, "all OK" } diff --git a/pkg/healthcheck/healthcheck_ipping.go b/pkg/healthcheck/healthcheck_ipping.go index caa923b..977c8cc 100644 --- a/pkg/healthcheck/healthcheck_ipping.go +++ b/pkg/healthcheck/healthcheck_ipping.go @@ -14,7 +14,7 @@ import ( ) // tcpConnect only does a tcp connection check -func ipPing(proto string, host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error) { +func ipPing(proto string, host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error, string) { errorcount := 0 errormsg := "" for i := 0; i < healthCheck.PINGpackets; i++ { @@ -28,10 +28,10 @@ func ipPing(proto string, host string, port int, sourceIP string, healthCheck He } if errorcount == healthCheck.PINGpackets { - return Offline, fmt.Errorf("%s ping lost 100%% packets: %s", proto, errormsg) + return Offline, fmt.Errorf("%s ping lost 100%% packets: %s", proto, errormsg), fmt.Sprintf("failed ping count") } - return Online, nil + return Online, nil, "OK" } func pingAddr(proto string, host string, port int, sourceIP string, seq int, dataSize int, timeout time.Duration) (bool, int, error) { diff --git a/pkg/healthcheck/healthcheck_ssh.go b/pkg/healthcheck/healthcheck_ssh.go index 3483fd2..b526174 100644 --- a/pkg/healthcheck/healthcheck_ssh.go +++ b/pkg/healthcheck/healthcheck_ssh.go @@ -8,7 +8,7 @@ import ( ) // tcpData does a simple tcp connect/reply check -func sshAuth(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error) { +func sshAuth(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error, string) { var sshConfig *ssh.ClientConfig if healthCheck.SSHPassword != "" { @@ -31,12 +31,12 @@ func sshAuth(host string, port int, sourceIP string, healthCheck HealthCheck) (S } tcpAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", host, port)) if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("failed to resolve to an address: %s:%d", host, port) } localAddr, errl := net.ResolveIPAddr("ip", sourceIP) if errl != nil { - return Offline, errl + return Offline, errl, fmt.Sprintf("failed to resolve to an ip adress: %s", sourceIP) } localTCPAddr := net.TCPAddr{ @@ -46,16 +46,16 @@ func sshAuth(host string, port int, sourceIP string, healthCheck HealthCheck) (S // Custom dialer with conn, err := net.DialTCP("tcp", &localTCPAddr, tcpAddr) if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("failed to dail from source: %+v target: %+v", localTCPAddr, *tcpAddr) } defer conn.Close() _, _, _, err = ssh.NewClientConn(conn, host, sshConfig) if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("failed to initiate ssh connection on %s with %+v", host, *sshConfig) } - return Online, nil + return Online, nil, "OK" } // publicKeyFile converts a string in to a ssh public key diff --git a/pkg/healthcheck/healthcheck_tcpconnect.go b/pkg/healthcheck/healthcheck_tcpconnect.go index 357a1a9..42ed52d 100644 --- a/pkg/healthcheck/healthcheck_tcpconnect.go +++ b/pkg/healthcheck/healthcheck_tcpconnect.go @@ -7,10 +7,10 @@ import ( ) // tcpConnect only does a tcp connection check -func tcpConnect(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error) { +func tcpConnect(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error, string) { localAddr, errl := net.ResolveIPAddr("ip", sourceIP) if errl != nil { - return Offline, errl + return Offline, errl, fmt.Sprintf("failed to resolve to an ip adress: %s", sourceIP) } localTCPAddr := net.TCPAddr{ @@ -26,9 +26,9 @@ func tcpConnect(host string, port int, sourceIP string, healthCheck HealthCheck) conn, err := dialer.Dial("tcp", fmt.Sprintf("%s:%d", host, port)) if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("failed to dial %s:%d", host, port) } conn.Close() - return Online, nil + return Online, nil, "OK" } diff --git a/pkg/healthcheck/healthcheck_tcpdata.go b/pkg/healthcheck/healthcheck_tcpdata.go index d87e38e..5ca7644 100644 --- a/pkg/healthcheck/healthcheck_tcpdata.go +++ b/pkg/healthcheck/healthcheck_tcpdata.go @@ -9,15 +9,16 @@ import ( ) // tcpData does a simple tcp connect/reply check -func tcpData(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error) { +func tcpData(host string, port int, sourceIP string, healthCheck HealthCheck) (Status, error, string) { + tcpAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", host, port)) if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("failed to resolve to an address: %s:%d", host, port) } localAddr, errl := net.ResolveIPAddr("ip", sourceIP) if errl != nil { - return Offline, errl + return Offline, errl, fmt.Sprintf("failed to resolve to an ip adress: %s", sourceIP) } localTCPAddr := net.TCPAddr{ @@ -27,7 +28,7 @@ func tcpData(host string, port int, sourceIP string, healthCheck HealthCheck) (S // Custom dialer with conn, err := net.DialTCP("tcp", &localTCPAddr, tcpAddr) if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("failed to dail from source: %+v target: %+v", localTCPAddr, *tcpAddr) } defer conn.Close() @@ -35,18 +36,18 @@ func tcpData(host string, port int, sourceIP string, healthCheck HealthCheck) (S fmt.Fprintf(conn, healthCheck.TCPRequest) r, err := regexp.Compile(healthCheck.TCPReply) if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("regex Compile failed on %s", healthCheck.TCPReply) } conn.SetReadDeadline(time.Now().Add(time.Duration(healthCheck.Timeout) * time.Second)) for { line, err := bufio.NewReader(conn).ReadString('\n') if err != nil { - return Offline, err + return Offline, err, fmt.Sprintf("failed with last input %s", line) } if r.MatchString(line) { - return Online, nil + return Online, nil, "OK" } } } diff --git a/pkg/healthcheck/healthcheck_worker.go b/pkg/healthcheck/healthcheck_worker.go index 03db72b..4bd8bdc 100644 --- a/pkg/healthcheck/healthcheck_worker.go +++ b/pkg/healthcheck/healthcheck_worker.go @@ -119,7 +119,7 @@ func (w *Worker) Start() { select { /* new check interval has reached */ case <-timer.C: - result, err := w.executeCheck() + result, err, _ := w.ExecuteCheck() // Send update if check result or error changes var checkerror string @@ -183,39 +183,40 @@ func (w *Worker) Stop() { } // executeCheck directs the check to the executioner and returns the result -func (w *Worker) executeCheck() (Status, error) { +func (w *Worker) ExecuteCheck() (Status, error, string) { var err error var result = Offline + var description string switch w.Check.Type { case "tcpconnect": - result, err = tcpConnect(w.IP, w.Port, w.SourceIP, w.Check) + result, err, description = tcpConnect(w.IP, w.Port, w.SourceIP, w.Check) case "tcpdata": - result, err = tcpData(w.IP, w.Port, w.SourceIP, w.Check) + result, err, description = tcpData(w.IP, w.Port, w.SourceIP, w.Check) case "ssh": - result, err = sshAuth(w.IP, w.Port, w.SourceIP, w.Check) + result, err, description = sshAuth(w.IP, w.Port, w.SourceIP, w.Check) case "httpget": - result, err = httpRequest("GET", w.IP, w.Port, w.SourceIP, w.Check) + result, err, description = httpRequest("GET", w.IP, w.Port, w.SourceIP, w.Check) case "httppost": - result, err = httpRequest("POST", w.IP, w.Port, w.SourceIP, w.Check) + result, err, description = httpRequest("POST", w.IP, w.Port, w.SourceIP, w.Check) case "icmpping": - result, err = ipPing("icmp", w.IP, 0, w.SourceIP, w.Check) + result, err, description = ipPing("icmp", w.IP, 0, w.SourceIP, w.Check) case "udpping": - result, err = ipPing("udp", w.IP, w.Port, w.SourceIP, w.Check) + result, err, description = ipPing("udp", w.IP, w.Port, w.SourceIP, w.Check) case "tcpping": - result, err = ipPing("tcp", w.IP, w.Port, w.SourceIP, w.Check) + result, err, description = ipPing("tcp", w.IP, w.Port, w.SourceIP, w.Check) default: result = Online } - return result, err + return result, err, description } func (w *Worker) filterWorker() (n Worker) { diff --git a/pkg/param/parameters.go b/pkg/param/parameters.go index 141095b..114b287 100644 --- a/pkg/param/parameters.go +++ b/pkg/param/parameters.go @@ -11,17 +11,18 @@ import ( // Config is the cmd parameter output type Config struct { - ConfigFile *string - PidFile *string - CheckGLB *bool - CheckConfig *bool - CheckBackend *bool - Debug *bool - Version *bool - BackendName *string - PoolName *string - DNSName *string - ClusterOnly *bool + ConfigFile *string + PidFile *string + CheckGLB *bool + CheckConfig *bool + CheckBackend *bool + CheckEndpoints *bool + Debug *bool + Version *bool + BackendName *string + PoolName *string + DNSName *string + ClusterOnly *bool } var ( @@ -32,17 +33,18 @@ var ( // Init needs to be called at the start of a program (used to be init, but the conflicts with go.1.13) func Init() { c := Config{ - ConfigFile: flag.String("config-file", "../../test/mercury.toml", "path to your mercury toml confg file"), - PidFile: flag.String("pid-file", "/run/mercury.pid", "path to your pid file"), - CheckGLB: flag.Bool("check-glb", false, "gives you a GLB report"), - CheckConfig: flag.Bool("check-config", false, "does a config check"), - CheckBackend: flag.Bool("check-backend", false, "gives you a Backend report"), - Debug: flag.Bool("debug", false, "force logging to debug mode"), - Version: flag.Bool("version", false, "display version"), - BackendName: flag.String("backend-name", "", "only check selected backend name"), - PoolName: flag.String("pool-name", "", "only check selected pool name"), - DNSName: flag.String("dns-name", "", "only check selected dns name"), - ClusterOnly: flag.Bool("cluster-only", false, "only check cluster"), + ConfigFile: flag.String("config-file", "../../test/mercury.toml", "path to your mercury toml confg file"), + PidFile: flag.String("pid-file", "/run/mercury.pid", "path to your pid file"), + CheckGLB: flag.Bool("check-glb", false, "gives you a GLB report"), + CheckConfig: flag.Bool("check-config", false, "does a config check"), + CheckBackend: flag.Bool("check-backend", false, "gives you a Backend report"), + CheckEndpoints: flag.Bool("check-endpoints", false, "runs a single check of all health checks of the endpoints"), + Debug: flag.Bool("debug", false, "force logging to debug mode"), + Version: flag.Bool("version", false, "display version"), + BackendName: flag.String("backend-name", "", "only check selected backend name"), + PoolName: flag.String("pool-name", "", "only check selected pool name"), + DNSName: flag.String("dns-name", "", "only check selected dns name"), + ClusterOnly: flag.Bool("cluster-only", false, "only check cluster"), } flag.Parse() config = &c