generated from kedacore/github-template
-
Notifications
You must be signed in to change notification settings - Fork 102
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add support to collect metrics using the OTEL library (#927)
* feat: Add support to collect metrics using either a prometheus compatible endpoint or by sending metrics to an OTEL http endpoint Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com> * fix: use UpDounCounter rather than gauge for pending request count and specify otel image for e2e tests Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com> --------- Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com>
- Loading branch information
Showing
29 changed files
with
1,228 additions
and
73 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| apiVersion: v1 | ||
| kind: Service | ||
| metadata: | ||
| name: interceptor-metrics | ||
| spec: | ||
| type: ClusterIP | ||
| ports: | ||
| - name: metrics | ||
| protocol: TCP | ||
| port: 2223 | ||
| targetPort: metrics |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| apiVersion: apps/v1 | ||
| kind: Deployment | ||
| metadata: | ||
| name: interceptor | ||
| spec: | ||
| replicas: 1 | ||
| template: | ||
| spec: | ||
| containers: | ||
| - name: interceptor | ||
| env: | ||
| - name: KEDA_HTTP_OTEL_PROM_EXPORTER_ENABLED | ||
| value: "true" | ||
| - name: KEDA_HTTP_OTEL_PROM_EXPORTER_PORT | ||
| value: "2223" | ||
| - name: KEDA_HTTP_OTEL_HTTP_EXPORTER_ENABLED | ||
| value: "true" | ||
| - name: KEDA_HTTP_OTEL_HTTP_COLLECTOR_ENDPOINT | ||
| value: "opentelemetry-collector.open-telemetry-system:4318" | ||
| - name: KEDA_HTTP_OTEL_HTTP_COLLECTOR_INSECURE | ||
| value: "true" | ||
| - name: KEDA_HTTP_OTEL_METRIC_EXPORT_INTERVAL | ||
| value: "1" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| apiVersion: kustomize.config.k8s.io/v1beta1 | ||
| kind: Kustomization | ||
| resources: | ||
| - deployment.yaml | ||
| - scaledobject.yaml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| apiVersion: keda.sh/v1alpha1 | ||
| kind: ScaledObject | ||
| metadata: | ||
| name: interceptor | ||
| spec: | ||
| minReplicaCount: 1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| # Configuring metrics for the KEDA HTTP Add-on interceptor proxy | ||
|
|
||
| ### Exportable metrics: | ||
| * **Pending request count** - the number of pending requests for a given host. | ||
| * **Total request count** - the total number of requests for a given host with method, path and response code attributes. | ||
|
|
||
| There are currently 2 supported methods for exposing metrics from the interceptor proxy service - via a Prometheus compatible metrics endpoint or by pushing metrics to a OTEL HTTP collector. | ||
|
|
||
| ### Configuring the Prometheus compatible metrics endpoint | ||
| When configured, the interceptor proxy can expose metrics on a Prometheus compatible endpoint. | ||
|
|
||
| This endpoint can be enabled by setting the `KEDA_HTTP_OTEL_PROM_EXPORTER_ENABLED` environment variable to `true` on the interceptor deployment (`true` by default) and by setting `KEDA_HTTP_OTEL_PROM_EXPORTER_PORT` to an unused port for the endpoint to be made avaialble on (`2223` by default). | ||
|
|
||
| ### Configuring the OTEL HTTP exporter | ||
| When configured, the interceptor proxy can export metrics to a OTEL HTTP collector. | ||
|
|
||
| The OTEL exporter can be enabled by setting the `KEDA_HTTP_OTEL_HTTP_EXPORTER_ENABLED` environment variable to `true` on the interceptor deployment (`false` by default). When enabled the `KEDA_HTTP_OTEL_HTTP_COLLECTOR_ENDPOINT` environment variable must also be configured so the exporter knows what collector to send the metrics to (e.g. opentelemetry-collector.open-telemetry-system:4318). | ||
|
|
||
| If the collector is exposed on a unsecured endpoint then you can set the `KEDA_HTTP_OTEL_HTTP_COLLECTOR_INSECURE` environment variable to `true` (`false` by default) which will disable client security on the exporter. | ||
|
|
||
| If you need to provide any headers such as authentication details in order to utilise your OTEL collector you can add them into the `KEDA_HTTP_OTEL_HTTP_HEADERS` environment variable. The frequency at which the metrics are exported can be configured by setting `KEDA_HTTP_OTEL_METRIC_EXPORT_INTERVAL` to the number of seconds you require between each export interval (`30` by default). |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| package config | ||
|
|
||
| import ( | ||
| "github.com/kelseyhightower/envconfig" | ||
| ) | ||
|
|
||
| // Metrics is the configuration for configuring metrics in the interceptor. | ||
| type Metrics struct { | ||
| // Sets whether or not to enable the Prometheus metrics exporter | ||
| OtelPrometheusExporterEnabled bool `envconfig:"KEDA_HTTP_OTEL_PROM_EXPORTER_ENABLED" default:"true"` | ||
| // Sets the port which the Prometheus compatible metrics endpoint should be served on | ||
| OtelPrometheusExporterPort int `envconfig:"KEDA_HTTP_OTEL_PROM_EXPORTER_PORT" default:"2223"` | ||
| // Sets whether or not to enable the OTEL metrics exporter | ||
| OtelHTTPExporterEnabled bool `envconfig:"KEDA_HTTP_OTEL_HTTP_EXPORTER_ENABLED" default:"false"` | ||
| // Sets the HTTP endpoint where metrics should be sent to | ||
| OtelHTTPCollectorEndpoint string `envconfig:"KEDA_HTTP_OTEL_HTTP_COLLECTOR_ENDPOINT" default:"localhost:4318"` | ||
| // Sets the OTLP headers required by the otel exporter | ||
| OtelHTTPHeaders string `envconfig:"KEDA_HTTP_OTEL_HTTP_HEADERS" default:""` | ||
| // Set the connection to the otel HTTP collector endpoint to use HTTP rather than HTTPS | ||
| OtelHTTPCollectorInsecure bool `envconfig:"KEDA_HTTP_OTEL_HTTP_COLLECTOR_INSECURE" default:"false"` | ||
| // Set the interval in seconds to export otel metrics to the configured collector endpoint | ||
| OtelMetricExportInterval int `envconfig:"KEDA_HTTP_OTEL_METRIC_EXPORT_INTERVAL" default:"30"` | ||
| } | ||
|
|
||
| // Parse parses standard configs using envconfig and returns a pointer to the | ||
| // newly created config. Returns nil and a non-nil error if parsing failed | ||
| func MustParseMetrics() *Metrics { | ||
| ret := new(Metrics) | ||
| envconfig.MustProcess("", ret) | ||
| return ret | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package metrics | ||
|
|
||
| import ( | ||
| "github.com/kedacore/http-add-on/interceptor/config" | ||
| ) | ||
|
|
||
| var ( | ||
| collectors []Collector | ||
| ) | ||
|
|
||
| const meterName = "keda-interceptor-proxy" | ||
|
|
||
| type Collector interface { | ||
| RecordRequestCount(method string, path string, responseCode int, host string) | ||
| RecordPendingRequestCount(host string, value int64) | ||
| } | ||
|
|
||
| func NewMetricsCollectors(metricsConfig *config.Metrics) { | ||
| if metricsConfig.OtelPrometheusExporterEnabled { | ||
| promometrics := NewPrometheusMetrics() | ||
| collectors = append(collectors, promometrics) | ||
| } | ||
|
|
||
| if metricsConfig.OtelHTTPExporterEnabled { | ||
| otelhttpmetrics := NewOtelMetrics(metricsConfig) | ||
| collectors = append(collectors, otelhttpmetrics) | ||
| } | ||
| } | ||
|
|
||
| func RecordRequestCount(method string, path string, responseCode int, host string) { | ||
| for _, collector := range collectors { | ||
| collector.RecordRequestCount(method, path, responseCode, host) | ||
| } | ||
| } | ||
|
|
||
| func RecordPendingRequestCount(host string, value int64) { | ||
| for _, collector := range collectors { | ||
| collector.RecordPendingRequestCount(host, value) | ||
| } | ||
| } |
Oops, something went wrong.