feat: Provide New Relic scaler

.github/workflows/main-build.yml

@@ -79,4 +79,7 @@ jobs:
           AZURE_DEVOPS_PAT: ${{ secrets.AZURE_DEVOPS_PAT }}
           AZURE_DEVOPS_POOL_NAME: ${{ secrets.AZURE_DEVOPS_POOL_NAME }}
           AZURE_DEVOPS_PROJECT: ${{ secrets.AZURE_DEVOPS_PROJECT }}
+          NEWRELIC_ACCOUNT_ID: ${{ secrets.NEWRELIC_ACCOUNT_ID}}
+          NEWRELIC_API_KEY: ${{ secrets.NEWRELIC_API_KEY}}
+          NEWRELIC_LICENSE: ${{ secrets.NEWRELIC_LICENSE}}
         run: make e2e-test

.github/workflows/nightly-e2e.yml

@@ -33,4 +33,7 @@ jobs:
           AZURE_DEVOPS_PAT: ${{ secrets.AZURE_DEVOPS_PAT }}
           AZURE_DEVOPS_POOL_NAME: ${{ secrets.AZURE_DEVOPS_POOL_NAME }}
           AZURE_DEVOPS_PROJECT: ${{ secrets.AZURE_DEVOPS_PROJECT }}
+          NEWRELIC_ACCOUNT_ID: ${{ secrets.NEWRELIC_ACCOUNT_ID}}
+          NEWRELIC_API_KEY: ${{ secrets.NEWRELIC_API_KEY}}
+          NEWRELIC_LICENSE: ${{ secrets.NEWRELIC_LICENSE}}
         run: make e2e-test

.github/workflows/pr-e2e.yml

@@ -79,6 +79,9 @@ jobs:
           AZURE_DEVOPS_PAT: ${{ secrets.AZURE_DEVOPS_PAT }}
           AZURE_DEVOPS_POOL_NAME: ${{ secrets.AZURE_DEVOPS_POOL_NAME }}
           AZURE_DEVOPS_PROJECT: ${{ secrets.AZURE_DEVOPS_PROJECT }}
+          NEWRELIC_ACCOUNT_ID: ${{ secrets.NEWRELIC_ACCOUNT_ID}}
+          NEWRELIC_API_KEY: ${{ secrets.NEWRELIC_API_KEY}}
+          NEWRELIC_LICENSE: ${{ secrets.NEWRELIC_LICENSE}}
           E2E_IMAGE_TAG: "pr-${{ steps.checkout.outputs.pr_num }}"
           TEST_CLUSTER_NAME: keda-pr-run
         run: |

CHANGELOG.md

@@ -23,6 +23,7 @@
 
 ### New
 
+- Add New Relic Scaler ([#2387](https://github.com/kedacore/keda/pull/2387))
 - TODO ([#XXX](https://github.com/kedacore/keda/pull/XXX))
 
 ### Improvements

go.mod

@@ -115,6 +115,7 @@ require (
 	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
@@ -128,7 +129,7 @@ require (
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-plugin v1.4.3 // indirect
-	github.com/hashicorp/go-retryablehttp v0.6.6 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/mlock v0.1.1 // indirect
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.1 // indirect
@@ -165,6 +166,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/newrelic/newrelic-client-go v0.68.3 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/oklog/run v1.0.0 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
@@ -173,11 +175,13 @@ require (
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cobra v1.1.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.0.2 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect

go.sum

@@ -263,6 +263,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP
 github.com/elastic/go-elasticsearch/v7 v7.15.1 h1:Wd8RLHb5D8xPBU8vGlnLXyflkso9G+rCmsXjqH8LLQQ=
 github.com/elastic/go-elasticsearch/v7 v7.15.1/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elazarl/goproxy v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.15.0+incompatible h1:8KpYO/Xl/ZudZs5RNOEhWMBY4hmzlZhhRd9cu+jrZP4=
@@ -454,6 +456,8 @@ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github/v27 v27.0.6/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
@@ -542,6 +546,8 @@ github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ3
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
 github.com/hashicorp/go-retryablehttp v0.6.6 h1:HJunrbHTDDbBb/ay4kxa1n+dLmttUlnP3V9oNE4hmsM=
 github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4=
+github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
 github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
@@ -735,6 +741,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/newrelic/newrelic-client-go v0.68.3 h1:PUGGKNakTV75+EG6tiiUvun8yqLzYr9ou9H9w0QiW+E=
+github.com/newrelic/newrelic-client-go v0.68.3/go.mod h1:VXjhsfui0rvhM9cVwnKwlidF8NbXlHZvh63ZKi6fImA=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
@@ -822,6 +830,7 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -888,6 +897,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
+github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
 github.com/tsenart/go-tsz v0.0.0-20180814232043-cdeb9e1e981e/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo=
 github.com/tsenart/vegeta/v12 v12.8.4/go.mod h1:ZiJtwLn/9M4fTPdMY7bdbIeyNeFVE8/AHbWFqCsUuho=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=

pkg/scalers/newrelic_scaler.go

@@ -0,0 +1,205 @@
+package scalers
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"log"
+	"strconv"
+
+	kedautil "github.com/kedacore/keda/v2/pkg/util"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/metrics/pkg/apis/external_metrics"
+
+	v2beta2 "k8s.io/api/autoscaling/v2beta2"
+	"k8s.io/apimachinery/pkg/labels"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"github.com/newrelic/newrelic-client-go/newrelic"
+	"github.com/newrelic/newrelic-client-go/pkg/nrdb"
+)
+
+const (
+	account    = "account"
+	queryKey   = "queryKey"
+	region     = "region"
+	metricName = "metricName"
+	nrql       = "nrql"
+	threshold  = "threshold"
+	noDataErr  = "noDataErr"
+)
+
+type newrelicScaler struct {
+	metadata *newrelicMetadata
+	nrClient *newrelic.NewRelic
+}
+
+type newrelicMetadata struct {
+	account     int
+	region      string
+	queryKey    string
+	metricName  string
+	noDataErr   bool
+	nrql        string
+	threshold   int
+	scalerIndex int
+}
+
+var newrelicLog = logf.Log.WithName("new-relic_scaler")
+
+func NewNewRelicScaler(config *ScalerConfig) (Scaler, error) {
+	meta, err := parseNewRelicMetadata(config)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing new-relic metadata: %s", err)
+	}
+
+	nrClient, err := newrelic.New(
+		newrelic.ConfigPersonalAPIKey(meta.queryKey),
+		newrelic.ConfigRegion(meta.region))
+
+	if err != nil {
+		log.Fatal("error initializing client:", err)
+	}
+
+	logMsg := fmt.Sprintf("Initializing New Relic Scaler (account %d in region %s)", meta.account, meta.region)
+
+	newrelicLog.Info(logMsg)
+
+	return &newrelicScaler{
+		metadata: meta,
+		nrClient: nrClient}, nil
+}
+
+func parseNewRelicMetadata(config *ScalerConfig) (*newrelicMetadata, error) {
+	meta := newrelicMetadata{}
+	var err error
+	if val, ok := config.TriggerMetadata[account]; ok && val != "" {
+		t, err := strconv.Atoi(val)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing %s: %s", account, err)
+		}
+		meta.account = t
+	} else {
+		return nil, fmt.Errorf("no %s given", account)
+	}
+
+	if val, ok := config.TriggerMetadata[nrql]; ok && val != "" {
+		meta.nrql = val
+	} else {
+		return nil, fmt.Errorf("no %s given", nrql)
+	}
+
+	meta.queryKey, err = GetFromAuthOrMeta(config, queryKey)
+	if err != nil {
+		return nil, fmt.Errorf("no %s given", queryKey)
+	}
+
+	if val, ok := config.TriggerMetadata[metricName]; ok && val != "" {
+		meta.metricName = val
+	}
+
+	meta.region, err = GetFromAuthOrMeta(config, region)
+	if err != nil {
+		meta.region = "US"
+		newrelicLog.Info("Using default 'US' region")
+	}
+
+	if val, ok := config.TriggerMetadata[threshold]; ok && val != "" {
+		t, err := strconv.Atoi(val)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing %s", threshold)
+		}
+		meta.threshold = t
+	} else {
+		return nil, fmt.Errorf("missing %s value", threshold)
+	}
+
+	// If Query Return an Empty Data , shall we treat it as an error or not
+	// default is NO error is returned when query result is empty/no data
+	if val, ok := config.TriggerMetadata[noDataErr]; ok {
+		noDataErr, err := strconv.ParseBool(val)
+		if err != nil {
+			return nil, fmt.Errorf("noDataErr has invalid value")
+		}
+		meta.noDataErr = noDataErr
+	} else {
+		meta.noDataErr = false
+	}
+	meta.scalerIndex = config.ScalerIndex
+	return &meta, nil
+}
+
+func (s *newrelicScaler) IsActive(ctx context.Context) (bool, error) {
+	val, err := s.ExecuteNewRelicQuery(ctx)
+	if err != nil {
+		newrelicLog.Error(err, "error executing newrelic query")
+		return false, err
+	}
+	return val > 0, nil
+}
+
+func (s *newrelicScaler) Close(context.Context) error {
+	return nil
+}
+
+func (s *newrelicScaler) ExecuteNewRelicQuery(ctx context.Context) (float64, error) {
+	nrdbQuery := nrdb.NRQL(s.metadata.nrql)
+	resp, err := s.nrClient.Nrdb.QueryWithContext(ctx, s.metadata.account, nrdbQuery)
+	if err != nil {
+		return 0, fmt.Errorf("error running NerdGraph query %s (%s)", s.metadata.nrql, err.Error())
+	}
+	// Only use the first result from the query, as the query should not be multi row
+	for _, v := range resp.Results[0] {
+		val, ok := v.(float64)
+		if ok {
+			return val, nil
+		}
+	}
+	if s.metadata.noDataErr {
+		return 0, fmt.Errorf("query return no results %s", s.metadata.nrql)
+	}
+	return 0, nil
+}
+
+func (s *newrelicScaler) GetMetrics(ctx context.Context, metricName string, metricSelector labels.Selector) ([]external_metrics.ExternalMetricValue, error) {
+	val, err := s.ExecuteNewRelicQuery(ctx)
+	if err != nil {
+		newrelicLog.Error(err, "error executing New Relic query")
+		return []external_metrics.ExternalMetricValue{}, err
+	}
+
+	metric := external_metrics.ExternalMetricValue{
+		MetricName: metricName,
+		Value:      *resource.NewQuantity(int64(val), resource.DecimalSI),
+		Timestamp:  metav1.Now(),
+	}
+
+	return append([]external_metrics.ExternalMetricValue{}, metric), nil
+}
+
+func (s *newrelicScaler) GetMetricSpecForScaling(context.Context) []v2beta2.MetricSpec {
+	targetMetricValue := resource.NewQuantity(int64(s.metadata.threshold), resource.DecimalSI)
+	var metricName string
+	if s.metadata.metricName == "" {
+		// TODO: Find a better way to generate a small string based on the nrql string, or maybe just use UUID?
+		hash := md5.Sum([]byte(s.metadata.nrql))
+		metricName = kedautil.NormalizeString(fmt.Sprintf("new-relic-%s", hex.EncodeToString(hash[:])))
+	} else {
+		metricName = kedautil.NormalizeString(fmt.Sprintf("new-relic-%s", s.metadata.metricName))
+	}
+	externalMetric := &v2beta2.ExternalMetricSource{
+		Metric: v2beta2.MetricIdentifier{
+			Name: GenerateMetricNameWithIndex(s.metadata.scalerIndex, metricName),
+		},
+		Target: v2beta2.MetricTarget{
+			Type:         v2beta2.AverageValueMetricType,
+			AverageValue: targetMetricValue,
+		},
+	}
+	metricSpec := v2beta2.MetricSpec{
+		External: externalMetric, Type: externalMetricType,
+	}
+	return []v2beta2.MetricSpec{metricSpec}
+}