From 13a2a5073fef88763a447436bca2433cdf463a2d Mon Sep 17 00:00:00 2001 From: GregoireW <24318548+GregoireW@users.noreply.github.com> Date: Thu, 16 Sep 2021 10:48:51 +0200 Subject: [PATCH] Add dynatrace provider Signed-off-by: GregoireW <24318548+GregoireW@users.noreply.github.com> --- charts/flagger/crds/crd.yaml | 1 + docs/gitbook/usage/metrics.md | 48 +++++++ kustomize/base/flagger/crd.yaml | 1 + pkg/metrics/providers/dynatrace.go | 181 ++++++++++++++++++++++++ pkg/metrics/providers/dynatrace_test.go | 170 ++++++++++++++++++++++ pkg/metrics/providers/factory.go | 2 + 6 files changed, 403 insertions(+) create mode 100644 pkg/metrics/providers/dynatrace.go create mode 100644 pkg/metrics/providers/dynatrace_test.go diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml index b0e148ad5..ebde3b211 100644 --- a/charts/flagger/crds/crd.yaml +++ b/charts/flagger/crds/crd.yaml @@ -1108,6 +1108,7 @@ spec: - cloudwatch - newrelic - graphite + - dynatrace address: description: API address of this provider type: string diff --git a/docs/gitbook/usage/metrics.md b/docs/gitbook/usage/metrics.md index 5199ed9df..2add89eab 100644 --- a/docs/gitbook/usage/metrics.md +++ b/docs/gitbook/usage/metrics.md @@ -561,3 +561,51 @@ spec: |> count() |> yield(name: "count") ``` + +## Dynatrace + +You can create custom metric checks using the Dynatrace provider. + +Create a secret with your Dynatrace token: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: dynatrace + namespace: istio-system +data: + dynatrace_token: ZHQwYz... +``` + +Dynatrace metric template example: + +```yaml +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: response-time-95pct + namespace: istio-system +spec: + provider: + type: dynatrace + address: https://xxxxxxxx.live.dynatrace.com + secretRef: + name: dynatrace + query: | + builtin:service.response.time:filter(eq(dt.entity.service,SERVICE-ABCDEFG0123456789)):percentile(95) +``` + +Reference the template in the canary analysis: + +```yaml + analysis: + metrics: + - name: "response-time-95pct" + templateRef: + name: response-time-95pct + namespace: istio-system + thresholdRange: + max: 1000 + interval: 1m +``` diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index b0e148ad5..ebde3b211 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -1108,6 +1108,7 @@ spec: - cloudwatch - newrelic - graphite + - dynatrace address: description: API address of this provider type: string diff --git a/pkg/metrics/providers/dynatrace.go b/pkg/metrics/providers/dynatrace.go new file mode 100644 index 000000000..7cdf5efde --- /dev/null +++ b/pkg/metrics/providers/dynatrace.go @@ -0,0 +1,181 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" +) + +// https://www.dynatrace.com/support/help/dynatrace-api/environment-api/metric-v2/get-all-metrics/ +const ( + dynatraceMetricsQueryPath = "/api/v2/metrics/query" + dynatraceValidationPath = "/api/v2/metrics?pageSize=1" + + dynatraceAPITokenSecretKey = "dynatrace_token" + dynatraceAuthorizationHeaderKey = "Authorization" + dynatraceAuthorizationHeaderType = "Api-Token" + + dynatraceDeltaMultiplierOnMetricInterval = 10 +) + +// DynatraceProvider executes dynatrace queries +type DynatraceProvider struct { + metricsQueryEndpoint string + apiValidationEndpoint string + + timeout time.Duration + token string + fromDelta int64 +} + +type dynatraceResponse struct { + Result []struct { + Data []struct { + Timestamps []int64 `json:"timestamps"` + Values []float64 `json:"values"` + } `json:"data"` + } `json:"result"` +} + +// NewDynatraceProvider takes a canary spec, a provider spec and the credentials map, and +// returns a Dynatrace client ready to execute queries against the API +func NewDynatraceProvider(metricInterval string, + provider flaggerv1.MetricTemplateProvider, + credentials map[string][]byte) (*DynatraceProvider, error) { + + address := provider.Address + if address == "" { + return nil, fmt.Errorf("dynatrace endpoint is not set") + } + + dt := DynatraceProvider{ + timeout: 5 * time.Second, + metricsQueryEndpoint: address + dynatraceMetricsQueryPath, + apiValidationEndpoint: address + dynatraceValidationPath, + } + + if b, ok := credentials[dynatraceAPITokenSecretKey]; ok { + dt.token = string(b) + } else { + return nil, fmt.Errorf("dynatrace credentials does not contain dynatrace_token") + } + + md, err := time.ParseDuration(metricInterval) + if err != nil { + return nil, fmt.Errorf("error parsing metric interval: %w", err) + } + + dt.fromDelta = int64(dynatraceDeltaMultiplierOnMetricInterval * md.Milliseconds()) + return &dt, nil +} + +// RunQuery executes the dynatrace query against DynatraceProvider.metricsQueryEndpoint +// and returns the the first result as float64 +func (p *DynatraceProvider) RunQuery(query string) (float64, error) { + + req, err := http.NewRequest("GET", p.metricsQueryEndpoint, nil) + if err != nil { + return 0, fmt.Errorf("error http.NewRequest: %w", err) + } + + req.Header.Set(dynatraceAuthorizationHeaderKey, fmt.Sprintf("%s %s", dynatraceAuthorizationHeaderType, p.token)) + + now := time.Now().Unix() * 1000 + q := req.URL.Query() + q.Add("metricSelector", query) + q.Add("resolution", "Inf") + q.Add("from", strconv.FormatInt(now-p.fromDelta, 10)) + q.Add("to", strconv.FormatInt(now, 10)) + req.URL.RawQuery = q.Encode() + + ctx, cancel := context.WithTimeout(req.Context(), p.timeout) + defer cancel() + r, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return 0, fmt.Errorf("request failed: %w", err) + } + + defer r.Body.Close() + b, err := io.ReadAll(r.Body) + if err != nil { + return 0, fmt.Errorf("error reading body: %w", err) + } + + if r.StatusCode != http.StatusOK { + return 0, fmt.Errorf("error response: %s: %w", string(b), err) + } + + var res dynatraceResponse + if err := json.Unmarshal(b, &res); err != nil { + return 0, fmt.Errorf("error unmarshaling result: %w, '%s'", err, string(b)) + } + + if len(res.Result) < 1 { + return 0, fmt.Errorf("invalid response: %s: %w", string(b), ErrNoValuesFound) + } + + data := res.Result[0].Data + if len(data) < 1 { + return 0, fmt.Errorf("invalid response: %s: %w", string(b), ErrNoValuesFound) + } + + vs := data[len(data)-1] + if len(vs.Values) < 1 { + return 0, fmt.Errorf("invalid response: %s: %w", string(b), ErrNoValuesFound) + } + + return vs.Values[0], nil +} + +// IsOnline calls the Dynatrace's metrics endpoint with token +// and returns an error if the endpoint fails +func (p *DynatraceProvider) IsOnline() (bool, error) { + req, err := http.NewRequest("GET", p.apiValidationEndpoint, nil) + if err != nil { + return false, fmt.Errorf("error http.NewRequest: %w", err) + } + + req.Header.Set(dynatraceAuthorizationHeaderKey, fmt.Sprintf("%s %s", dynatraceAuthorizationHeaderType, p.token)) + + ctx, cancel := context.WithTimeout(req.Context(), p.timeout) + defer cancel() + r, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return false, fmt.Errorf("request failed: %w", err) + } + + defer r.Body.Close() + + b, err := io.ReadAll(r.Body) + if err != nil { + return false, fmt.Errorf("error reading body: %w", err) + } + + if r.StatusCode != http.StatusOK { + return false, fmt.Errorf("error response: %s", string(b)) + } + + return true, nil +} diff --git a/pkg/metrics/providers/dynatrace_test.go b/pkg/metrics/providers/dynatrace_test.go new file mode 100644 index 000000000..9d2e4c9a8 --- /dev/null +++ b/pkg/metrics/providers/dynatrace_test.go @@ -0,0 +1,170 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" +) + +func TestNewDynatraceProvider(t *testing.T) { + token := "token" + cs := map[string][]byte{ + dynatraceAPITokenSecretKey: []byte(token), + } + + mi := "100s" + md, err := time.ParseDuration(mi) + require.NoError(t, err) + + _, err = NewDynatraceProvider("100s", flaggerv1.MetricTemplateProvider{}, cs) + require.Error(t, err) + + dp, err := NewDynatraceProvider("100s", flaggerv1.MetricTemplateProvider{Address: "https://mySampleEnv.live.dynatrace.com"}, cs) + require.NoError(t, err) + assert.Equal(t, "https://mySampleEnv.live.dynatrace.com/api/v2/metrics/query", dp.metricsQueryEndpoint) + assert.Equal(t, "https://mySampleEnv.live.dynatrace.com/api/v2/metrics?pageSize=1", dp.apiValidationEndpoint) + assert.Equal(t, int64(md.Milliseconds()*dynatraceDeltaMultiplierOnMetricInterval), dp.fromDelta) + assert.Equal(t, token, dp.token) +} + +func TestDynatraceProvider_RunQuery(t *testing.T) { + token := "token" + t.Run("ok", func(t *testing.T) { + expected := 1.11111 + eq := `builtin:host.cpu.usage:filter(eq(Host,HOST-0990886B7D39FE29))` + now := time.Now().Unix() * 1000 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + aq := r.URL.Query().Get("metricSelector") + assert.Equal(t, eq, aq) + assert.Equal(t, "Api-Token token", r.Header.Get(dynatraceAuthorizationHeaderKey)) + + from, err := strconv.ParseInt(r.URL.Query().Get("from"), 10, 64) + if assert.NoError(t, err) { + assert.Less(t, from, now) + } + + to, err := strconv.ParseInt(r.URL.Query().Get("to"), 10, 64) + if assert.NoError(t, err) { + assert.GreaterOrEqual(t, to, now) + } + + resultTemplate := ` +{ + "totalCount": 1, + "nextPageKey": null, + "result": [ + { + "metricId": "builtin:host.cpu.usage", + "data": [ + { + "dimensions": [ + "HOST-0990886B7D39FE29" + ], + "timestamps": [ + 1589455320000 + ], + "values": [ + %f + ] + } + ] + } + ] +} +` + + json := fmt.Sprintf(resultTemplate, expected) + w.Write([]byte(json)) + })) + defer ts.Close() + + dp, err := NewDynatraceProvider("1m", + flaggerv1.MetricTemplateProvider{Address: ts.URL}, + map[string][]byte{ + dynatraceAPITokenSecretKey: []byte(token), + }, + ) + require.NoError(t, err) + + f, err := dp.RunQuery(eq) + require.NoError(t, err) + assert.Equal(t, expected, f) + }) + + t.Run("no values", func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json := fmt.Sprintf(`{"series": [{"pointlist": []}]}`) + w.Write([]byte(json)) + })) + defer ts.Close() + + dp, err := NewDynatraceProvider("1m", + flaggerv1.MetricTemplateProvider{Address: ts.URL}, + map[string][]byte{ + dynatraceAPITokenSecretKey: []byte(token), + }, + ) + require.NoError(t, err) + _, err = dp.RunQuery("") + require.True(t, errors.Is(err, ErrNoValuesFound)) + }) +} + +func TestDynatraceProvider_IsOnline(t *testing.T) { + for _, c := range []struct { + code int + errExpected bool + }{ + {code: http.StatusOK, errExpected: false}, + {code: http.StatusUnauthorized, errExpected: true}, + } { + t.Run(fmt.Sprintf("%d", c.code), func(t *testing.T) { + token := "token" + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "Api-Token token", r.Header.Get(dynatraceAuthorizationHeaderKey)) + w.WriteHeader(c.code) + })) + defer ts.Close() + + dp, err := NewDynatraceProvider("1m", + flaggerv1.MetricTemplateProvider{Address: ts.URL}, + map[string][]byte{ + dynatraceAPITokenSecretKey: []byte(token), + }, + ) + require.NoError(t, err) + + _, err = dp.IsOnline() + if c.errExpected { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/pkg/metrics/providers/factory.go b/pkg/metrics/providers/factory.go index 91031779e..db5bd9f5b 100644 --- a/pkg/metrics/providers/factory.go +++ b/pkg/metrics/providers/factory.go @@ -42,6 +42,8 @@ func (factory Factory) Provider( return NewStackDriverProvider(provider, credentials) case "influxdb": return NewInfluxdbProvider(provider, credentials) + case "dynatrace": + return NewDynatraceProvider(metricInterval, provider, credentials) default: return NewPrometheusProvider(provider, credentials) }