From 886e62f4c10f528bf012637ed27ab89a8567d835 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan <4194920+tigrannajaryan@users.noreply.github.com> Date: Tue, 2 Jul 2019 22:27:41 -0400 Subject: [PATCH] Implement Prometheus factory and config V2 (#89) Github issue: https://github.com/open-telemetry/opentelemetry-service/issues/38 Testing done: - make && make otelsvc - Run otelsvc with the following config and make sure Prometheus can scrape : receivers: opencensus: port: 55678 prometheus: config: scrape_configs: - job_name: 'demo' scrape_interval: 5s zpages: port: 55679 exporters: prometheus: endpoint: "127.0.0.1:8889" pipelines: metrics: receivers: [prometheus] exporters: [prometheus] --- .../app/builder/receivers_builder.go | 2 +- exporter/prometheusexporter/config.go | 35 +++++++ exporter/prometheusexporter/config_test.go | 57 +++++++++++ exporter/prometheusexporter/factory.go | 98 +++++++++++++++++++ exporter/prometheusexporter/factory_test.go | 79 +++++++++++++++ .../prometheusexporter/prometheus_test.go | 83 +++++++--------- .../prometheusexporter/testdata/config.yaml | 21 ++++ 7 files changed, 329 insertions(+), 46 deletions(-) create mode 100644 exporter/prometheusexporter/config.go create mode 100644 exporter/prometheusexporter/config_test.go create mode 100644 exporter/prometheusexporter/factory.go create mode 100644 exporter/prometheusexporter/factory_test.go create mode 100644 exporter/prometheusexporter/testdata/config.yaml diff --git a/cmd/occollector/app/builder/receivers_builder.go b/cmd/occollector/app/builder/receivers_builder.go index ae97d192292b..eea094971335 100644 --- a/cmd/occollector/app/builder/receivers_builder.go +++ b/cmd/occollector/app/builder/receivers_builder.go @@ -203,7 +203,7 @@ func (rb *ReceiversBuilder) attachReceiverToPipelines( dataType.GetString(), dataType.GetString()) } - return fmt.Errorf("cannot create receiver %s", config.Name()) + return fmt.Errorf("cannot create receiver %s: %s", config.Name(), err.Error()) } rb.logger.Info("Receiver is enabled.", diff --git a/exporter/prometheusexporter/config.go b/exporter/prometheusexporter/config.go new file mode 100644 index 000000000000..d2204762ae3a --- /dev/null +++ b/exporter/prometheusexporter/config.go @@ -0,0 +1,35 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusexporter + +import ( + prometheus_golang "github.com/prometheus/client_golang/prometheus" + + "github.com/open-telemetry/opentelemetry-service/models" +) + +// ConfigV2 defines configuration for Prometheus exporter. +type ConfigV2 struct { + models.ExporterSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct. + + // The address on which the Prometheus scrape handler will be run on. + Endpoint string `mapstructure:"endpoint"` + + // Namespace if set, exports metrics under the provided value. + Namespace string `mapstructure:"namespace"` + + // ConstLabels are values that are applied for every exported metric. + ConstLabels prometheus_golang.Labels `mapstructure:"const_labels"` +} diff --git a/exporter/prometheusexporter/config_test.go b/exporter/prometheusexporter/config_test.go new file mode 100644 index 000000000000..b6b12c638bb9 --- /dev/null +++ b/exporter/prometheusexporter/config_test.go @@ -0,0 +1,57 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusexporter + +import ( + "path" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/open-telemetry/opentelemetry-service/configv2" + "github.com/open-telemetry/opentelemetry-service/factories" + "github.com/open-telemetry/opentelemetry-service/models" +) + +var _ = configv2.RegisterTestFactories() + +func TestLoadConfig(t *testing.T) { + factory := factories.GetExporterFactory(typeStr) + + config, err := configv2.LoadConfigFile(t, path.Join(".", "testdata", "config.yaml")) + + require.NoError(t, err) + require.NotNil(t, config) + + e0 := config.Exporters["prometheus"] + assert.Equal(t, e0, factory.CreateDefaultConfig()) + + e1 := config.Exporters["prometheus/2"] + assert.Equal(t, e1, + &ConfigV2{ + ExporterSettings: models.ExporterSettings{ + NameVal: "prometheus/2", + TypeVal: "prometheus", + Enabled: true, + }, + Endpoint: "1.2.3.4:1234", + Namespace: "test-space", + ConstLabels: map[string]string{ + "label1": "value1", + "another label": "spaced value", + }, + }) +} diff --git a/exporter/prometheusexporter/factory.go b/exporter/prometheusexporter/factory.go new file mode 100644 index 000000000000..8412abb20825 --- /dev/null +++ b/exporter/prometheusexporter/factory.go @@ -0,0 +1,98 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusexporter + +import ( + "net" + "net/http" + "strings" + + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-service/consumer" + "github.com/open-telemetry/opentelemetry-service/factories" + "github.com/open-telemetry/opentelemetry-service/models" + "github.com/orijtech/prometheus-go-metrics-exporter" +) + +var _ = factories.RegisterExporterFactory(&exporterFactory{}) + +const ( + // The value of "type" key in configuration. + typeStr = "prometheus" +) + +// exporterFactory is the factory for Prometheus exporter. +type exporterFactory struct { +} + +// Type gets the type of the Exporter config created by this factory. +func (f *exporterFactory) Type() string { + return typeStr +} + +// CreateDefaultConfig creates the default configuration for exporter. +func (f *exporterFactory) CreateDefaultConfig() models.Exporter { + return &ConfigV2{ + ExporterSettings: models.ExporterSettings{ + TypeVal: typeStr, + NameVal: typeStr, + }, + ConstLabels: map[string]string{}, + } +} + +// CreateTraceExporter creates a trace exporter based on this config. +func (f *exporterFactory) CreateTraceExporter(logger *zap.Logger, config models.Exporter) (consumer.TraceConsumer, factories.StopFunc, error) { + return nil, nil, models.ErrDataTypeIsNotSupported +} + +// CreateMetricsExporter creates a metrics exporter based on this config. +func (f *exporterFactory) CreateMetricsExporter(logger *zap.Logger, cfg models.Exporter) (consumer.MetricsConsumer, factories.StopFunc, error) { + pcfg := cfg.(*ConfigV2) + + addr := strings.TrimSpace(pcfg.Endpoint) + if addr == "" { + return nil, nil, errBlankPrometheusAddress + } + + opts := prometheus.Options{ + Namespace: pcfg.Namespace, + ConstLabels: pcfg.ConstLabels, + } + pe, err := prometheus.New(opts) + if err != nil { + return nil, nil, err + } + + ln, err := net.Listen("tcp", addr) + if err != nil { + return nil, nil, err + } + + // The Prometheus metrics exporter has to run on the provided address + // as a server that'll be scraped by Prometheus. + mux := http.NewServeMux() + mux.Handle("/metrics", pe) + + srv := &http.Server{Handler: mux} + go func() { + _ = srv.Serve(ln) + }() + + pexp := &prometheusExporter{exporter: pe} + + return pexp, ln.Close, nil +} diff --git a/exporter/prometheusexporter/factory_test.go b/exporter/prometheusexporter/factory_test.go new file mode 100644 index 000000000000..6845851f1b97 --- /dev/null +++ b/exporter/prometheusexporter/factory_test.go @@ -0,0 +1,79 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusexporter + +import ( + "testing" + + "github.com/open-telemetry/opentelemetry-service/models" + + "go.uber.org/zap" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/open-telemetry/opentelemetry-service/factories" +) + +func TestCreateDefaultConfig(t *testing.T) { + factory := factories.GetExporterFactory(typeStr) + require.NotNil(t, factory) + + cfg := factory.CreateDefaultConfig() + assert.NotNil(t, cfg, "failed to create default config") +} + +func TestCreateTraceExporter(t *testing.T) { + factory := factories.GetExporterFactory(typeStr) + cfg := factory.CreateDefaultConfig() + + _, _, err := factory.CreateTraceExporter(zap.NewNop(), cfg) + assert.Error(t, err, models.ErrDataTypeIsNotSupported) +} + +func TestCreateMetricsExporter(t *testing.T) { + const defaultTestEndPoint = "127.0.0.1:55678" + tests := []struct { + name string + config ConfigV2 + mustFail bool + }{ + { + name: "NoEndpoint", + config: ConfigV2{ + Endpoint: "", + }, + mustFail: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + factory := factories.GetExporterFactory(typeStr) + consumer, stopFunc, err := factory.CreateMetricsExporter(zap.NewNop(), &tt.config) + + if tt.mustFail { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + assert.NotNil(t, consumer) + assert.NotNil(t, stopFunc) + + err = stopFunc() + assert.Nil(t, err) + } + }) + } +} diff --git a/exporter/prometheusexporter/prometheus_test.go b/exporter/prometheusexporter/prometheus_test.go index c26afcb5bdcf..ec791ebb01f8 100644 --- a/exporter/prometheusexporter/prometheus_test.go +++ b/exporter/prometheusexporter/prometheus_test.go @@ -22,35 +22,43 @@ import ( "testing" "github.com/golang/protobuf/ptypes/timestamp" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + "github.com/open-telemetry/opentelemetry-service/data" + "github.com/open-telemetry/opentelemetry-service/factories" viperutils "github.com/open-telemetry/opentelemetry-service/internal/config/viperutils" ) func TestPrometheusExporter(t *testing.T) { tests := []struct { - config string + config *ConfigV2 wantErr string }{ { - config: ` -prometheus: - namespace: "test" - const_labels: { - "foo": "bar", - "code": "one" - } - address: ":8999" -`, + config: &ConfigV2{ + Namespace: "test", + ConstLabels: map[string]string{ + "foo": "bar", + "code": "one", + }, + Endpoint: ":8999", + }, + }, + { + config: &ConfigV2{}, + wantErr: "expecting a non-blank address to run the Prometheus metrics handler", }, } + factory := factories.GetExporterFactory(typeStr) for i, tt := range tests { // Run it a few times to ensure that shutdowns exit cleanly. for j := 0; j < 3; j++ { - v, _ := viperutils.ViperFromYAMLBytes([]byte(tt.config)) - tes, mes, doneFns, err := PrometheusExportersFromViper(v) + consumer, stopFunc, err := factory.CreateMetricsExporter(zap.NewNop(), tt.config) + if tt.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Errorf("#%d iteration #%d: Unexpected error: %v Wanted: %v", i, j, err, tt.wantErr) @@ -58,18 +66,12 @@ prometheus: continue } + assert.NotNil(t, consumer) + if err != nil { t.Errorf("#%d iteration #%d: unexpected parse error: %v", i, j, err) } - if len(tes) != 0 { - t.Errorf("#%d iteration #%d: Unexpectedly got back %d > 0 trace exporters", i, j, len(tes)) - } - if len(mes) == 0 { - t.Errorf("#%d iteration #%d: Unexpectedly got back 0 metrics exporters", i, j) - } - for _, doneFn := range doneFns { - doneFn() - } + stopFunc() } } } @@ -94,30 +96,22 @@ prometheus:`) } func TestPrometheusExporter_endToEnd(t *testing.T) { - config := []byte(` -prometheus: - namespace: "test" - const_labels: { - "foo": "bar", - "code": "one" - } - address: ":7777" -`) + config := &ConfigV2{ + Namespace: "test", + ConstLabels: map[string]string{ + "foo": "bar", + "code": "one", + }, + Endpoint: ":7777", + } - v, _ := viperutils.ViperFromYAMLBytes([]byte(config)) - _, mes, doneFns, err := PrometheusExportersFromViper(v) - defer func() { - for _, doneFn := range doneFns { - doneFn() - } - }() + factory := factories.GetExporterFactory(typeStr) + consumer, stopFunc, err := factory.CreateMetricsExporter(zap.NewNop(), config) + assert.Nil(t, err) - if err != nil { - t.Fatalf("Unexpected parse error: %v", err) - } - if len(mes) == 0 { - t.Fatal("Unexpectedly got back 0 metrics exporters") - } + defer stopFunc() + + assert.NotNil(t, consumer) var metric1 = &metricspb.Metric{ MetricDescriptor: &metricspb.MetricDescriptor{ @@ -153,8 +147,7 @@ prometheus: }, }, } - me := mes[0] - me.ConsumeMetricsData(context.Background(), data.MetricsData{Metrics: []*metricspb.Metric{metric1}}) + consumer.ConsumeMetricsData(context.Background(), data.MetricsData{Metrics: []*metricspb.Metric{metric1}}) res, err := http.Get("http://localhost:7777/metrics") if err != nil { diff --git a/exporter/prometheusexporter/testdata/config.yaml b/exporter/prometheusexporter/testdata/config.yaml new file mode 100644 index 000000000000..6e2dc1a084b8 --- /dev/null +++ b/exporter/prometheusexporter/testdata/config.yaml @@ -0,0 +1,21 @@ +receivers: + examplereceiver: + +processors: + exampleprocessor: + +exporters: + prometheus: + prometheus/2: + enabled: true + endpoint: "1.2.3.4:1234" + namespace: test-space + const_labels: + label1: value1 + "another label": spaced value + +pipelines: + traces: + receivers: [examplereceiver] + processors: [exampleprocessor] + exporters: [prometheus]