Fixes and improvements

.travis.yml

@@ -1 +1,4 @@
 language: go
+
+go:
+- 1.13

Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:1.13-alpine as builder
+
+ENV CGO_ENABLED=0
+
+COPY . /app
+WORKDIR /app
+RUN apk --no-cache add make && make build
+
+FROM alpine:latest
+
+WORKDIR /app
+
+COPY --from=builder /app/builds/checkup /usr/local/bin/checkup
+
+ENTRYPOINT ["checkup"]

Makefile

@@ -0,0 +1,15 @@
+.PHONY: all build test docker
+
+all: build test
+
+DOCKER_IMAGE := checkup
+
+build:
+	mkdir -p builds/
+	go build -o builds/ ./cmd/...
+
+test:
+	go test -race -count=1 -v ./...
+
+docker:
+	docker build --no-cache . -t $(DOCKER_IMAGE)

checkup.go

@@ -175,6 +175,8 @@ func (c Checkup) MarshalJSON() ([]byte, error) {
 			}
 			var typeName string
 			switch ch.(type) {
+			case ExecChecker:
+				typeName = "exec"
 			case HTTPChecker:
 				typeName = "http"
 			case TCPChecker:
@@ -287,6 +289,13 @@ func (c *Checkup) UnmarshalJSON(b []byte) error {
 	// assertions with the help of the type information
 	for i, t := range types.Checkers {
 		switch t.Type {
+		case "exec":
+			var checker ExecChecker
+			err = json.Unmarshal(raw.Checkers[i], &checker)
+			if err != nil {
+				return err
+			}
+			c.Checkers = append(c.Checkers, checker)
 		case "http":
 			var checker HTTPChecker
 			err = json.Unmarshal(raw.Checkers[i], &checker)

checkup_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"sync"
 	"testing"
 	"time"
 )
@@ -71,6 +72,8 @@ func TestCheckAndStoreEvery(t *testing.T) {
 	time.Sleep(170 * time.Millisecond)
 	ticker.Stop()
 
+	f.Lock()
+	defer f.Unlock()
 	if got, want := f.checked, 3; got != want {
 		t.Errorf("Expected %d checks while sleeping, had: %d", want, got)
 	}
@@ -193,6 +196,8 @@ func TestJSON(t *testing.T) {
 var errTest = errors.New("i'm an error")
 
 type fake struct {
+	sync.Mutex
+
 	returnErr  bool
 	checked    int
 	stored     []Result
@@ -201,6 +206,9 @@ type fake struct {
 }
 
 func (f *fake) Check() (Result, error) {
+	f.Lock()
+	defer f.Unlock()
+
 	f.checked++
 	r := Result{Timestamp: time.Now().UTC().UnixNano()}
 	if f.returnErr {
@@ -210,6 +218,9 @@ func (f *fake) Check() (Result, error) {
 }
 
 func (f *fake) Store(results []Result) error {
+	f.Lock()
+	defer f.Unlock()
+
 	f.stored = results
 	if f.returnErr {
 		return errTest
@@ -218,11 +229,17 @@ func (f *fake) Store(results []Result) error {
 }
 
 func (f *fake) Maintain() error {
+	f.Lock()
+	defer f.Unlock()
+
 	f.maintained++
 	return nil
 }
 
 func (f *fake) Notify(results []Result) error {
+	f.Lock()
+	defer f.Unlock()
+
 	f.notified++
 	return nil
 }

execchecker.go

@@ -0,0 +1,168 @@
+package checkup
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// ExecChecker implements a Checker by running programs with os.Exec.
+type ExecChecker struct {
+	// Name is the name of the endpoint.
+	Name string `json:"name"`
+
+	// Command is the main program entrypoint.
+	Command string `json:"command"`
+
+	// Arguments are individual program parameters.
+	Arguments []string `json:"arguments,omitempty"`
+
+	// ThresholdRTT is the maximum round trip time to
+	// allow for a healthy endpoint. If non-zero and a
+	// request takes longer than ThresholdRTT, the
+	// endpoint will be considered unhealthy. Note that
+	// this duration includes any in-between network
+	// latency.
+	ThresholdRTT time.Duration `json:"threshold_rtt,omitempty"`
+
+	// MustContain is a string that the response body
+	// must contain in order to be considered up.
+	// NOTE: If set, the entire response body will
+	// be consumed, which has the potential of using
+	// lots of memory and slowing down checks if the
+	// response body is large.
+	MustContain string `json:"must_contain,omitempty"`
+
+	// MustNotContain is a string that the response
+	// body must NOT contain in order to be considered
+	// up. If both MustContain and MustNotContain are
+	// set, they are and-ed together. NOTE: If set,
+	// the entire response body will be consumed, which
+	// has the potential of using lots of memory and
+	// slowing down checks if the response body is large.
+	MustNotContain string `json:"must_not_contain,omitempty"`
+
+	// Raise is a string that tells us if we should throw
+	// a hard error ("error" - the default), or if we should
+	// just mark something as degraded ("warn" or "warning").
+	Raise string `json:"raise,omitempty"`
+
+	// Attempts is how many requests the client will
+	// make to the endpoint in a single check.
+	Attempts int `json:"attempts,omitempty"`
+
+	// AttemptSpacing spaces out each attempt in a check
+	// by this duration to avoid hitting a remote too
+	// quickly in succession. By default, no waiting
+	// occurs between attempts.
+	AttemptSpacing time.Duration `json:"attempt_spacing,omitempty"`
+}
+
+// Check performs checks using c according to its configuration.
+// An error is only returned if there is a configuration error.
+func (c ExecChecker) Check() (Result, error) {
+	if c.Attempts < 1 {
+		c.Attempts = 1
+	}
+
+	result := Result{
+		Title:     c.Name,
+		Endpoint:  c.Command,
+		Timestamp: Timestamp(),
+	}
+
+	result.Times = c.doChecks()
+
+	return c.conclude(result), nil
+}
+
+// doChecks executes command and returns each attempt.
+func (c ExecChecker) doChecks() Attempts {
+	checks := make(Attempts, c.Attempts)
+	for i := 0; i < c.Attempts; i++ {
+		start := time.Now()
+
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+
+		command := exec.CommandContext(ctx, c.Command, c.Arguments...)
+		output, err := command.CombinedOutput()
+
+		checks[i].RTT = time.Since(start)
+
+		if err != nil {
+			stringify := func(s string) string {
+				if strings.TrimSpace(s) == "" {
+					return "empty"
+				}
+				return s
+			}
+			checks[i].Error = fmt.Sprintf("Error: %s\nOutput: %s\n", err.Error(), stringify(string(output)))
+			continue
+		}
+
+		if err := c.checkDown(string(output)); err != nil {
+			checks[i].Error = err.Error()
+		}
+
+		if c.AttemptSpacing > 0 {
+			time.Sleep(c.AttemptSpacing)
+		}
+	}
+	return checks
+}
+
+// conclude takes the data in result from the attempts and
+// computes remaining values needed to fill out the result.
+// It detects degraded (high-latency) responses and makes
+// the conclusion about the result's status.
+func (c ExecChecker) conclude(result Result) Result {
+	result.ThresholdRTT = c.ThresholdRTT
+
+	warning := c.Raise == "warn" || c.Raise == "warning"
+
+	// Check errors (down)
+	for i := range result.Times {
+		if result.Times[i].Error != "" {
+			if warning {
+				result.Notice = result.Times[i].Error
+				result.Degraded = true
+				return result
+			}
+			result.Down = true
+			return result
+		}
+	}
+
+	// Check round trip time (degraded)
+	if c.ThresholdRTT > 0 {
+		stats := result.ComputeStats()
+		if stats.Median > c.ThresholdRTT {
+			result.Notice = fmt.Sprintf("median round trip time exceeded threshold (%s)", c.ThresholdRTT)
+			result.Degraded = true
+			return result
+		}
+	}
+
+	result.Healthy = true
+	return result
+}
+
+// checkDown checks whether the endpoint is down based on resp and
+// the configuration of c. It returns a non-nil error if down.
+// Note that it does not check for degraded response.
+func (c ExecChecker) checkDown(body string) error {
+	// Check response body
+	if c.MustContain == "" && c.MustNotContain == "" {
+		return nil
+	}
+	if c.MustContain != "" && !strings.Contains(body, c.MustContain) {
+		return fmt.Errorf("response does not contain '%s'", c.MustContain)
+	}
+	if c.MustNotContain != "" && strings.Contains(body, c.MustNotContain) {
+		return fmt.Errorf("response contains '%s'", c.MustNotContain)
+	}
+	return nil
+}

execchecker_test.go

@@ -0,0 +1,39 @@
+package checkup
+
+import (
+	"testing"
+)
+
+func TestExecChecker(t *testing.T) {
+
+	assert := func(ok bool, format string, args ...interface{}) {
+		if !ok {
+			t.Fatalf(format, args...)
+		}
+	}
+
+	command := "testdata/exec.sh"
+
+	// check non-zero exit code
+	{
+		testName := "Non-zero exit"
+		hc := ExecChecker{Name: testName, Command: command, Arguments: []string{"1", testName}, Attempts: 2}
+
+		result, err := hc.Check()
+		assert(err == nil, "expected no error, got %v, %#v", err, result)
+		assert(result.Title == testName, "expected result.Title == %s, got %s", testName, result.Title)
+		assert(result.Down == true, "expected result.Down = true, got %v", result.Down)
+	}
+
+	// check zero exit code
+	{
+		testName := "Non-zero exit"
+		hc := ExecChecker{Name: testName, Command: command, Arguments: []string{"0", testName}, Attempts: 2}
+
+		result, err := hc.Check()
+		t.Logf("%#v", result)
+		assert(err == nil, "expected no error, got %v, %#v", err, result)
+		assert(result.Title == testName, "expected result.Title == %s, got %s", testName, result.Title)
+		assert(result.Down == false, "expected result.Down = false, got %v", result.Down)
+	}
+}

go.mod

@@ -0,0 +1,21 @@
+module github.com/sourcegraph/checkup
+
+go 1.13
+
+require (
+	github.com/ashwanthkumar/slack-go-webhook v0.0.0-20200209025033-430dd4e66960
+	github.com/aws/aws-sdk-go v1.30.2
+	github.com/elazarl/goproxy v0.0.0-20200315184450-1f3cb6622dad // indirect
+	github.com/fatih/color v1.9.0
+	github.com/google/go-github v17.0.0+incompatible
+	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/jmoiron/sqlx v1.2.0
+	github.com/lib/pq v1.3.0
+	github.com/mattn/go-sqlite3 v2.0.3+incompatible
+	github.com/miekg/dns v1.1.29
+	github.com/parnurzeal/gorequest v0.2.16 // indirect
+	github.com/smartystreets/goconvey v1.6.4 // indirect
+	github.com/spf13/cobra v0.0.7
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	moul.io/http2curl v1.0.0 // indirect
+)