Add New Relic backend

README.md

@@ -1,6 +1,6 @@
 # Buildkite Agent Metrics
 
-A command-line tool for collecting [Buildkite](https://buildkite.com/) agent metrics, focusing on enabling auto-scaling. Currently [AWS Cloudwatch](http://aws.amazon.com/cloudwatch/), [StatsD](https://github.com/etsy/statsd), [Prometheus](https://prometheus.io) and [Stackdriver](https://cloud.google.com/stackdriver/) are supported.
+A command-line tool for collecting [Buildkite](https://buildkite.com/) agent metrics, focusing on enabling auto-scaling. Currently [AWS Cloudwatch](http://aws.amazon.com/cloudwatch/), [StatsD](https://github.com/etsy/statsd), [Prometheus](https://prometheus.io), [Stackdriver](https://cloud.google.com/stackdriver/) and [New Relic](https://newrelic.com/products/insights) are supported.
 
 [![Build status](https://badge.buildkite.com/80d04fcde3a306bef44e77aadb1f1ffdc20ebb3c8f1f585a60.svg)](https://buildkite.com/buildkite/buildkite-agent-metrics)
 

backend/backends.go

@@ -6,3 +6,8 @@ import "github.com/buildkite/buildkite-agent-metrics/collector"
 type Backend interface {
 	Collect(r *collector.Result) error
 }
+
+// Closer is an interface for backends that need to dispose of resources
+type Closer interface {
+	Close() error
+}

backend/newrelic.go

@@ -0,0 +1,74 @@
+package backend
+
+import (
+	"log"
+	"time"
+
+	"github.com/buildkite/buildkite-agent-metrics/collector"
+	newrelic "github.com/newrelic/go-agent"
+)
+
+const newRelicConnectionTimeout = time.Second * 30
+
+// NewRelicBackend sends metrics to New Relic Insights
+type NewRelicBackend struct {
+	client newrelic.Application
+}
+
+// NewNewRelicBackend returns a backend for New Relic
+// Where appName is your desired application name in New Relic
+//   and licenseKey is your New Relic license key
+func NewNewRelicBackend(appName string, licenseKey string) (*NewRelicBackend, error) {
+	config := newrelic.NewConfig(appName, licenseKey)
+	app, err := newrelic.NewApplication(config)
+	if err != nil {
+		return nil, err
+	}
+
+	// Waiting for connection is essential or no data will make it during short-lived execution (e.g. Lambda)
+	err = app.WaitForConnection(newRelicConnectionTimeout)
+	if err != nil {
+		return nil, err
+	}
+
+	return &NewRelicBackend{
+		client: app,
+	}, nil
+}
+
+// Collect metrics
+func (nr *NewRelicBackend) Collect(r *collector.Result) error {
+	// Publish event for each queue
+	for name, c := range r.Queues {
+		data := toCustomEvent(name, c)
+		err := nr.client.RecordCustomEvent("BuildkiteQueueMetrics", data)
+		if err != nil {
+			return err
+		}
+
+		nr.client.RecordCustomEvent("queue_agent_metrics", data)
+	}
+
+	return nil
+}
+
+// toCustomEvent converts a map of metrics to a valid New Relic event body
+func toCustomEvent(queueName string, queueMetrics map[string]int) map[string]interface{} {
+	eventData := map[string]interface{}{
+		"Queue": queueName,
+	}
+
+	for k, v := range queueMetrics {
+		eventData[k] = v
+	}
+
+	return eventData
+}
+
+// Close by shutting down NR client
+func (nr *NewRelicBackend) Close() error {
+	nr.client.Shutdown(newRelicConnectionTimeout)
+	log.Printf("Disposed New Relic client")
+
+	return nil
+}

backend/newrelic_test.go

@@ -0,0 +1,66 @@
+package backend
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestToCustomEvent(t *testing.T) {
+	tcs := []struct {
+		queueName string                 // input queue
+		metrics   map[string]int         // input metrics
+		expected  map[string]interface{} // output shaped data
+	}{
+		// test 1 partial data
+		{
+			queueName: "partial-data-test",
+			metrics: map[string]int{
+				"BusyAgentCount":      0,
+				"BusyAgentPercentage": 0,
+				"IdleAgentCount":      3,
+				"TotalAgentCount":     3,
+				"RunningJobsCount":    0,
+			},
+			expected: map[string]interface{}{
+				"Queue":               "partial-data-test",
+				"BusyAgentCount":      0,
+				"BusyAgentPercentage": 0,
+				"IdleAgentCount":      3,
+				"TotalAgentCount":     3,
+				"RunningJobsCount":    0,
+			},
+		},
+		// test 2 complete data
+		{
+			queueName: "complete-data-test",
+			metrics: map[string]int{
+				"BusyAgentCount":      2,
+				"BusyAgentPercentage": 20,
+				"IdleAgentCount":      8,
+				"TotalAgentCount":     10,
+				"RunningJobsCount":    2,
+				"ScheduledJobsCount":  0,
+				"WaitingJobsCount":    0,
+			},
+			expected: map[string]interface{}{
+				"Queue":               "complete-data-test",
+				"BusyAgentCount":      2,
+				"BusyAgentPercentage": 20,
+				"IdleAgentCount":      8,
+				"TotalAgentCount":     10,
+				"RunningJobsCount":    2,
+				"ScheduledJobsCount":  0,
+				"WaitingJobsCount":    0,
+			},
+		},
+	}
+
+	for n, tc := range tcs {
+		got := toCustomEvent(tc.queueName, tc.metrics)
+
+		if !reflect.DeepEqual(got, tc.expected) {
+			t.Errorf("toCustomEvent test #%d failed, result %+v did not equal expected %+v", n, got, tc.expected)
+		}
+	}
+
+}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/aws/aws-sdk-go v1.15.66
 	github.com/golang/protobuf v1.2.0
 	github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
+	github.com/newrelic/go-agent v2.7.0+incompatible
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f
 	google.golang.org/genproto v0.0.0-20190401181712-f467c93bbac2

go.sum

@@ -86,6 +86,8 @@ github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/newrelic/go-agent v2.7.0+incompatible h1:T5tJ9nNY1bXBfLUTCEZRuBLPT0f9+mE1jd4EaNoN5Zs=
+github.com/newrelic/go-agent v2.7.0+incompatible/go.mod h1:a8Fv1b/fYhFSReoTU6HDkTYIMZeSVNffmoS726Y0LzQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=

lambda/main.go

@@ -71,14 +71,23 @@ func Handler(ctx context.Context, evt json.RawMessage) (string, error) {
 
 	var b backend.Backend
 	var err error
-	if backendOpt == "statsd" {
+	switch strings.ToLower(backendOpt) {
+	case "statsd":
 		statsdHost := os.Getenv("STATSD_HOST")
-		statsdTags := strings.ToLower(os.Getenv("STATSD_TAGS")) == "true"
+		statsdTags := strings.EqualFold(os.Getenv("STATSD_TAGS"), "true")
 		b, err = backend.NewStatsDBackend(statsdHost, statsdTags)
 		if err != nil {
 			return "", err
 		}
-	} else {
+	case "newrelic":
+		nrAppName := os.Getenv("NEWRELIC_APP_NAME")
+		nrLicenseKey := os.Getenv("NEWRELIC_LICENSE_KEY")
+		b, err = backend.NewNewRelicBackend(nrAppName, nrLicenseKey)
+		if err != nil {
+			fmt.Printf("Error starting New Relic client: %v\n", err)
+			os.Exit(1)
+		}
+	default:
 		dimensions, err := backend.ParseCloudWatchDimensions(clwDimensions)
 		if err != nil {
 			return "", err
@@ -98,6 +107,11 @@ func Handler(ctx context.Context, evt json.RawMessage) (string, error) {
 		return "", err
 	}
 
+	original, ok := b.(backend.Closer)
+	if ok {
+		original.Close()
+	}
+
 	log.Printf("Finished in %s", time.Now().Sub(t))
 
 	// Store the next acceptable poll time in global state

main.go

@@ -36,6 +36,8 @@ func main() {
 		clwRegion      = flag.String("cloudwatch-region", "", "AWS Region to connect to, defaults to $AWS_REGION or us-east-1")
 		clwDimensions  = flag.String("cloudwatch-dimensions", "", "Cloudwatch dimensions to index metrics under, in the form of Key=Value, Other=Value")
 		gcpProjectID   = flag.String("stackdriver-projectid", "", "Specify Stackdriver Project ID")
+		nrAppName      = flag.String("newrelic-app-name", "", "New Relic application name for metric events")
+		nrLicenseKey   = flag.String("newrelic-license-key", "", "New Relic license key for publishing events")
 
 		// filters
 		queue = flag.String("queue", "", "Only include a specific queue")
@@ -53,6 +55,7 @@ func main() {
 		os.Exit(1)
 	}
 
+	var err error
 	switch strings.ToLower(*backendOpt) {
 	case "cloudwatch":
 		region := *clwRegion
@@ -68,7 +71,6 @@ func main() {
 		}
 		bk = backend.NewCloudWatchBackend(region, dimensions)
 	case "statsd":
-		var err error
 		bk, err = backend.NewStatsDBackend(*statsdHost, *statsdTags)
 		if err != nil {
 			fmt.Printf("Error starting StatsD, err: %v\n", err)
@@ -77,14 +79,19 @@ func main() {
 	case "prometheus":
 		bk = backend.NewPrometheusBackend(*prometheusPath, *prometheusAddr)
 	case "stackdriver":
-		var err error
 		bk, err = backend.NewStackDriverBackend(*gcpProjectID)
 		if err != nil {
 			fmt.Printf("Error starting Stackdriver backend, err: %v\n", err)
 			os.Exit(1)
 		}
+	case "newrelic":
+		bk, err = backend.NewNewRelicBackend(*nrAppName, *nrLicenseKey)
+		if err != nil {
+			fmt.Printf("Error starting New Relic client: %v\n", err)
+			os.Exit(1)
+		}
 	default:
-		fmt.Println("Must provide a supported backend: cloudwatch, statsd, prometheus")
+		fmt.Println("Must provide a supported backend: cloudwatch, statsd, prometheus, stackdriver, newrelic")
 		os.Exit(1)
 	}