From 695fdff448ea9398c470d946df94fce5181e4a6a Mon Sep 17 00:00:00 2001 From: Pablo Baeyens Date: Thu, 10 Mar 2022 12:44:50 +0100 Subject: [PATCH 1/4] [exporter/datadog] Remove reference to `Config` on `GetHost` --- exporter/datadogexporter/internal/metadata/host.go | 7 +++---- .../datadogexporter/internal/metadata/host_test.go | 11 +++-------- .../datadogexporter/internal/metadata/metadata.go | 2 +- exporter/datadogexporter/metrics_exporter.go | 2 +- exporter/datadogexporter/traces_exporter.go | 2 +- 5 files changed, 9 insertions(+), 15 deletions(-) diff --git a/exporter/datadogexporter/internal/metadata/host.go b/exporter/datadogexporter/internal/metadata/host.go index 4b7ff1b61008..64f58aaf90af 100644 --- a/exporter/datadogexporter/internal/metadata/host.go +++ b/exporter/datadogexporter/internal/metadata/host.go @@ -17,7 +17,6 @@ package metadata // import "github.com/open-telemetry/opentelemetry-collector-co import ( "go.uber.org/zap" - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/metadata/ec2" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/metadata/system" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/metadata/valid" @@ -30,9 +29,9 @@ import ( // 2. Cache // 3. EC2 instance metadata // 4. System -func GetHost(logger *zap.Logger, cfg *config.Config) string { - if cfg.Hostname != "" { - return cfg.Hostname +func GetHost(logger *zap.Logger, configHostname string) string { + if configHostname != "" { + return configHostname } if cacheVal, ok := cache.Cache.Get(cache.CanonicalHostnameKey); ok { diff --git a/exporter/datadogexporter/internal/metadata/host_test.go b/exporter/datadogexporter/internal/metadata/host_test.go index 96edd0686edc..e40c3177af5b 100644 --- a/exporter/datadogexporter/internal/metadata/host_test.go +++ b/exporter/datadogexporter/internal/metadata/host_test.go @@ -22,7 +22,6 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/zap" - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/utils/cache" ) @@ -34,15 +33,11 @@ func TestHost(t *testing.T) { // if the cache key is already set. cache.Cache.Delete(cache.CanonicalHostnameKey) - host := GetHost(logger, &config.Config{ - TagsConfig: config.TagsConfig{Hostname: "test-host"}, - }) + host := GetHost(logger, "test-host") assert.Equal(t, host, "test-host") // config.Config.Hostname does not get stored in the cache - host = GetHost(logger, &config.Config{ - TagsConfig: config.TagsConfig{Hostname: "test-host-2"}, - }) + host = GetHost(logger, "test-host-2") assert.Equal(t, host, "test-host-2") // Disable EC2 Metadata service to prevent fetching hostname from there, @@ -53,7 +48,7 @@ func TestHost(t *testing.T) { require.NoError(t, err) defer os.Setenv(awsEc2MetadataDisabled, curr) - host = GetHost(logger, &config.Config{}) + host = GetHost(logger, "") osHostname, err := os.Hostname() require.NoError(t, err) // TODO: Investigate why the returned host contains more data on github actions. diff --git a/exporter/datadogexporter/internal/metadata/metadata.go b/exporter/datadogexporter/internal/metadata/metadata.go index a29028933933..057d3df3f7d6 100644 --- a/exporter/datadogexporter/internal/metadata/metadata.go +++ b/exporter/datadogexporter/internal/metadata/metadata.go @@ -123,7 +123,7 @@ func metadataFromAttributes(attrs pdata.AttributeMap) *HostMetadata { func fillHostMetadata(params component.ExporterCreateSettings, cfg *config.Config, hm *HostMetadata) { // Could not get hostname from attributes if hm.InternalHostname == "" { - hostname := GetHost(params.Logger, cfg) + hostname := GetHost(params.Logger, cfg.Hostname) hm.InternalHostname = hostname hm.Meta.Hostname = hostname } diff --git a/exporter/datadogexporter/metrics_exporter.go b/exporter/datadogexporter/metrics_exporter.go index 7541ca76f062..17e9a3eab4d3 100644 --- a/exporter/datadogexporter/metrics_exporter.go +++ b/exporter/datadogexporter/metrics_exporter.go @@ -55,7 +55,7 @@ type hostProvider struct { } func (p *hostProvider) Hostname(context.Context) (string, error) { - return metadata.GetHost(p.logger, p.cfg), nil + return metadata.GetHost(p.logger, p.cfg.Hostname), nil } // translatorFromConfig creates a new metrics translator from the exporter config. diff --git a/exporter/datadogexporter/traces_exporter.go b/exporter/datadogexporter/traces_exporter.go index f362356e149c..07c9b3a94249 100644 --- a/exporter/datadogexporter/traces_exporter.go +++ b/exporter/datadogexporter/traces_exporter.go @@ -127,7 +127,7 @@ func (exp *traceExporter) pushTraceData( // convert traces to datadog traces and group trace payloads by env // we largely apply the same logic as the serverless implementation, simplified a bit // https://github.com/DataDog/datadog-serverless-functions/blob/f5c3aedfec5ba223b11b76a4239fcbf35ec7d045/aws/logs_monitoring/trace_forwarder/cmd/trace/main.go#L61-L83 - fallbackHost := metadata.GetHost(exp.params.Logger, exp.cfg) + fallbackHost := metadata.GetHost(exp.params.Logger, exp.cfg.Hostname) ddTraces, ms := convertToDatadogTd(td, fallbackHost, exp.cfg, exp.denylister, exp.params.BuildInfo) // group the traces by env to reduce the number of flushes From ac59bafbc28d2a034d4c15fc2a20784322ea3b6e Mon Sep 17 00:00:00 2001 From: Pablo Baeyens Date: Thu, 10 Mar 2022 12:46:44 +0100 Subject: [PATCH 2/4] [exporter/datadog] Remove reference to Config on ProcessMetrics --- exporter/datadogexporter/internal/metrics/utils.go | 4 +--- .../datadogexporter/internal/metrics/utils_test.go | 13 +------------ exporter/datadogexporter/metrics_exporter.go | 2 +- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/exporter/datadogexporter/internal/metrics/utils.go b/exporter/datadogexporter/internal/metrics/utils.go index 1e9d55353089..019785b1a827 100644 --- a/exporter/datadogexporter/internal/metrics/utils.go +++ b/exporter/datadogexporter/internal/metrics/utils.go @@ -20,8 +20,6 @@ import ( "go.opentelemetry.io/collector/component" "gopkg.in/zorkian/go-datadog-api.v2" - - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config" ) type MetricDataType string @@ -94,7 +92,7 @@ func DefaultMetrics(exporterType string, hostname string, timestamp uint64, buil // ProcessMetrics adds the hostname to the metric and prefixes it with the "otel" // namespace as the Datadog backend expects -func ProcessMetrics(ms []datadog.Metric, cfg *config.Config) { +func ProcessMetrics(ms []datadog.Metric) { addNamespace(ms, otelNamespacePrefix) } diff --git a/exporter/datadogexporter/internal/metrics/utils_test.go b/exporter/datadogexporter/internal/metrics/utils_test.go index 794274fc2b60..d196d2ea78c5 100644 --- a/exporter/datadogexporter/internal/metrics/utils_test.go +++ b/exporter/datadogexporter/internal/metrics/utils_test.go @@ -19,10 +19,8 @@ import ( "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/component" - "go.uber.org/zap" "gopkg.in/zorkian/go-datadog-api.v2" - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/utils/cache" ) @@ -82,15 +80,6 @@ func TestProcessMetrics(t *testing.T) { // Reset hostname cache cache.Cache.Flush() - cfg := &config.Config{ - // Global tags should be ignored and sent as metadata - TagsConfig: config.TagsConfig{ - Env: "test_env", - Tags: []string{"key:val"}, - }, - } - cfg.Sanitize(zap.NewNop()) - ms := []datadog.Metric{ NewGauge( "metric_name", @@ -106,7 +95,7 @@ func TestProcessMetrics(t *testing.T) { ), } - ProcessMetrics(ms, cfg) + ProcessMetrics(ms) assert.Equal(t, "metric_name", *ms[0].Metric) assert.ElementsMatch(t, diff --git a/exporter/datadogexporter/metrics_exporter.go b/exporter/datadogexporter/metrics_exporter.go index 17e9a3eab4d3..f23181c7bb54 100644 --- a/exporter/datadogexporter/metrics_exporter.go +++ b/exporter/datadogexporter/metrics_exporter.go @@ -174,7 +174,7 @@ func (exp *metricsExporter) PushMetricsData(ctx context.Context, md pdata.Metric return fmt.Errorf("failed to map metrics: %w", err) } ms, sl := consumer.All(pushTime, exp.params.BuildInfo) - metrics.ProcessMetrics(ms, exp.cfg) + metrics.ProcessMetrics(ms) err = nil if len(ms) > 0 { From ee8b160b8095a96c427bf293362ce209ed35ebd8 Mon Sep 17 00:00:00 2001 From: Pablo Baeyens Date: Thu, 10 Mar 2022 13:58:51 +0100 Subject: [PATCH 3/4] [exporter/datadog] Decouple metadata pusher configuration from main configuration --- exporter/datadogexporter/factory.go | 4 +- exporter/datadogexporter/hostmetadata.go | 34 ++++++++++++++++ .../internal/metadata/config.go | 39 +++++++++++++++++++ .../internal/metadata/metadata.go | 31 +++++++-------- .../internal/metadata/metadata_test.go | 39 ++++++++++--------- .../datadogexporter/internal/utils/http.go | 6 +-- exporter/datadogexporter/metrics_exporter.go | 4 +- exporter/datadogexporter/trace_connection.go | 2 +- exporter/datadogexporter/traces_exporter.go | 2 +- 9 files changed, 116 insertions(+), 45 deletions(-) create mode 100644 exporter/datadogexporter/hostmetadata.go create mode 100644 exporter/datadogexporter/internal/metadata/config.go diff --git a/exporter/datadogexporter/factory.go b/exporter/datadogexporter/factory.go index 7825e3c2fe7e..f27c33fd53d4 100644 --- a/exporter/datadogexporter/factory.go +++ b/exporter/datadogexporter/factory.go @@ -129,7 +129,7 @@ func createMetricsExporter( if md.ResourceMetrics().Len() > 0 { attrs = md.ResourceMetrics().At(0).Resource().Attributes() } - go metadata.Pusher(ctx, set, cfg, attrs) + go metadata.Pusher(ctx, set, newMetadataConfigfromConfig(cfg), attrs) }) return nil } @@ -189,7 +189,7 @@ func createTracesExporter( if td.ResourceSpans().Len() > 0 { attrs = td.ResourceSpans().At(0).Resource().Attributes() } - go metadata.Pusher(ctx, set, cfg, attrs) + go metadata.Pusher(ctx, set, newMetadataConfigfromConfig(cfg), attrs) }) return nil } diff --git a/exporter/datadogexporter/hostmetadata.go b/exporter/datadogexporter/hostmetadata.go new file mode 100644 index 000000000000..3cf49ca3b921 --- /dev/null +++ b/exporter/datadogexporter/hostmetadata.go @@ -0,0 +1,34 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datadogexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter" + +import ( + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/metadata" +) + +// newMetadataConfigfromConfig creates a new metadata pusher config from the main config. +func newMetadataConfigfromConfig(cfg *config.Config) metadata.PusherConfig { + return metadata.PusherConfig{ + ConfigHostname: cfg.Hostname, + ConfigTags: cfg.GetHostTags(), + MetricsEndpoint: cfg.Metrics.Endpoint, + APIKey: cfg.API.Key, + UseResourceMetadata: cfg.UseResourceMetadata, + InsecureSkipVerify: cfg.TLSSetting.InsecureSkipVerify, + TimeoutSettings: cfg.TimeoutSettings, + RetrySettings: cfg.RetrySettings, + } +} diff --git a/exporter/datadogexporter/internal/metadata/config.go b/exporter/datadogexporter/internal/metadata/config.go new file mode 100644 index 000000000000..b3a5796fa1c0 --- /dev/null +++ b/exporter/datadogexporter/internal/metadata/config.go @@ -0,0 +1,39 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/metadata" + +import ( + "go.opentelemetry.io/collector/exporter/exporterhelper" +) + +// PusherConfig is the configuration for the metadata pusher goroutine. +type PusherConfig struct { + // ConfigHosthame is the hostname set in the configuration of the exporter (empty if unset). + ConfigHostname string + // ConfigTags are the tags set in the configuration of the exporter (empty if unset). + ConfigTags []string + // MetricsEndpoint is the metrics endpoint. + MetricsEndpoint string + // APIKey is the API key set in configuration. + APIKey string + // UseResourceMetadata is the value of 'use_resource_metadata' on the top-level configuration. + UseResourceMetadata bool + // InsecureSkipVerify is the value of `tls.insecure_skip_verify` on the configuration. + InsecureSkipVerify bool + // TimeoutSettings of exporter. + TimeoutSettings exporterhelper.TimeoutSettings + // RetrySettings of exporter. + RetrySettings exporterhelper.RetrySettings +} diff --git a/exporter/datadogexporter/internal/metadata/metadata.go b/exporter/datadogexporter/internal/metadata/metadata.go index 057d3df3f7d6..da43acde9ce6 100644 --- a/exporter/datadogexporter/internal/metadata/metadata.go +++ b/exporter/datadogexporter/internal/metadata/metadata.go @@ -27,7 +27,6 @@ import ( conventions "go.opentelemetry.io/collector/model/semconv/v1.6.1" "go.uber.org/zap" - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/metadata/ec2" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/metadata/system" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/model/attributes" @@ -120,10 +119,10 @@ func metadataFromAttributes(attrs pdata.AttributeMap) *HostMetadata { return hm } -func fillHostMetadata(params component.ExporterCreateSettings, cfg *config.Config, hm *HostMetadata) { +func fillHostMetadata(params component.ExporterCreateSettings, mcfg PusherConfig, hm *HostMetadata) { // Could not get hostname from attributes if hm.InternalHostname == "" { - hostname := GetHost(params.Logger, cfg.Hostname) + hostname := GetHost(params.Logger, mcfg.ConfigHostname) hm.InternalHostname = hostname hm.Meta.Hostname = hostname } @@ -132,7 +131,7 @@ func fillHostMetadata(params component.ExporterCreateSettings, cfg *config.Confi // since it does not come from OTEL conventions hm.Flavor = params.BuildInfo.Command hm.Version = params.BuildInfo.Version - hm.Tags.OTel = append(hm.Tags.OTel, cfg.GetHostTags()...) + hm.Tags.OTel = append(hm.Tags.OTel, mcfg.ConfigTags...) // EC2 data was not set from attributes if hm.Meta.EC2Hostname == "" { @@ -149,19 +148,19 @@ func fillHostMetadata(params component.ExporterCreateSettings, cfg *config.Confi } } -func pushMetadata(cfg *config.Config, params component.ExporterCreateSettings, metadata *HostMetadata) error { +func pushMetadata(mcfg PusherConfig, params component.ExporterCreateSettings, metadata *HostMetadata) error { if metadata.Meta.Hostname == "" { // if the hostname is empty, don't send metadata; we don't need it. params.Logger.Debug("Skipping host metadata since the hostname is empty") return nil } - path := cfg.Metrics.TCPAddr.Endpoint + "/intake" + path := mcfg.MetricsEndpoint + "/intake" buf, _ := json.Marshal(metadata) req, _ := http.NewRequest(http.MethodPost, path, bytes.NewBuffer(buf)) - utils.SetDDHeaders(req.Header, params.BuildInfo, cfg.API.Key) + utils.SetDDHeaders(req.Header, params.BuildInfo, mcfg.APIKey) utils.SetExtraHeaders(req.Header, utils.JSONHeaders) - client := utils.NewHTTPClient(cfg.TimeoutSettings, cfg.LimitedHTTPClientSettings) + client := utils.NewHTTPClient(mcfg.TimeoutSettings, mcfg.InsecureSkipVerify) resp, err := client.Do(req) if err != nil { @@ -181,11 +180,11 @@ func pushMetadata(cfg *config.Config, params component.ExporterCreateSettings, m return nil } -func pushMetadataWithRetry(retrier *utils.Retrier, params component.ExporterCreateSettings, cfg *config.Config, hostMetadata *HostMetadata) { +func pushMetadataWithRetry(retrier *utils.Retrier, params component.ExporterCreateSettings, mcfg PusherConfig, hostMetadata *HostMetadata) { params.Logger.Debug("Sending host metadata payload", zap.Any("payload", hostMetadata)) err := retrier.DoWithRetries(context.Background(), func(context.Context) error { - return pushMetadata(cfg, params, hostMetadata) + return pushMetadata(mcfg, params, hostMetadata) }) if err != nil { @@ -197,12 +196,12 @@ func pushMetadataWithRetry(retrier *utils.Retrier, params component.ExporterCrea } // Pusher pushes host metadata payloads periodically to Datadog intake -func Pusher(ctx context.Context, params component.ExporterCreateSettings, cfg *config.Config, attrs pdata.AttributeMap) { +func Pusher(ctx context.Context, params component.ExporterCreateSettings, mcfg PusherConfig, attrs pdata.AttributeMap) { // Push metadata every 30 minutes ticker := time.NewTicker(30 * time.Minute) defer ticker.Stop() defer params.Logger.Debug("Shut down host metadata routine") - retrier := utils.NewRetrier(params.Logger, cfg.RetrySettings, scrub.NewScrubber()) + retrier := utils.NewRetrier(params.Logger, mcfg.RetrySettings, scrub.NewScrubber()) // Get host metadata from resources and fill missing info using our exporter. // Currently we only retrieve it once but still send the same payload @@ -212,20 +211,20 @@ func Pusher(ctx context.Context, params component.ExporterCreateSettings, cfg *c // do not change over time. If this ever changes `hostMetadata` // *must* be deep copied before calling `fillHostMetadata`. hostMetadata := &HostMetadata{Meta: &Meta{}, Tags: &HostTags{}} - if cfg.UseResourceMetadata { + if mcfg.UseResourceMetadata { hostMetadata = metadataFromAttributes(attrs) } - fillHostMetadata(params, cfg, hostMetadata) + fillHostMetadata(params, mcfg, hostMetadata) // Run one first time at startup - pushMetadataWithRetry(retrier, params, cfg, hostMetadata) + pushMetadataWithRetry(retrier, params, mcfg, hostMetadata) for { select { case <-ctx.Done(): return case <-ticker.C: // Send host metadata - pushMetadataWithRetry(retrier, params, cfg, hostMetadata) + pushMetadataWithRetry(retrier, params, mcfg, hostMetadata) } } } diff --git a/exporter/datadogexporter/internal/metadata/metadata_test.go b/exporter/datadogexporter/internal/metadata/metadata_test.go index c85ab84a5c29..1d618c5df5b1 100644 --- a/exporter/datadogexporter/internal/metadata/metadata_test.go +++ b/exporter/datadogexporter/internal/metadata/metadata_test.go @@ -29,7 +29,6 @@ import ( "go.opentelemetry.io/collector/component/componenttest" conventions "go.opentelemetry.io/collector/model/semconv/v1.6.1" - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/model/attributes" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/model/attributes/azure" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/testutils" @@ -67,14 +66,13 @@ func TestFillHostMetadata(t *testing.T) { params := componenttest.NewNopExporterCreateSettings() params.BuildInfo = mockBuildInfo - cfg := &config.Config{TagsConfig: config.TagsConfig{ - Hostname: "hostname", - Env: "prod", - Tags: []string{"key1:tag1", "key2:tag2"}, - }} + mcfg := PusherConfig{ + ConfigHostname: "hostname", + ConfigTags: []string{"key1:tag1", "key2:tag2", "env:prod"}, + } metadata := &HostMetadata{Meta: &Meta{}, Tags: &HostTags{}} - fillHostMetadata(params, cfg, metadata) + fillHostMetadata(params, mcfg, metadata) assert.Equal(t, metadata.InternalHostname, "hostname") assert.Equal(t, metadata.Flavor, "otelcontribcol") @@ -88,7 +86,7 @@ func TestFillHostMetadata(t *testing.T) { Tags: &HostTags{}, } - fillHostMetadata(params, cfg, metadataWithVals) + fillHostMetadata(params, mcfg, metadataWithVals) assert.Equal(t, metadataWithVals.InternalHostname, "my-custom-hostname") assert.Equal(t, metadataWithVals.Flavor, "otelcontribcol") assert.Equal(t, metadataWithVals.Version, "1.0") @@ -156,7 +154,9 @@ func TestMetadataFromAttributes(t *testing.T) { } func TestPushMetadata(t *testing.T) { - cfg := &config.Config{API: config.APIConfig{Key: "apikey"}} + mcfg := PusherConfig{ + APIKey: "apikey", + } handler := http.NewServeMux() handler.HandleFunc("/intake", func(w http.ResponseWriter, r *http.Request) { @@ -174,29 +174,30 @@ func TestPushMetadata(t *testing.T) { ts := httptest.NewServer(handler) defer ts.Close() - cfg.Metrics.Endpoint = ts.URL + mcfg.MetricsEndpoint = ts.URL - err := pushMetadata(cfg, mockExporterCreateSettings, &mockMetadata) + err := pushMetadata(mcfg, mockExporterCreateSettings, &mockMetadata) require.NoError(t, err) } func TestFailPushMetadata(t *testing.T) { - cfg := &config.Config{API: config.APIConfig{Key: "apikey"}} - + mcfg := PusherConfig{ + APIKey: "apikey", + } handler := http.NewServeMux() handler.Handle("/intake", http.NotFoundHandler()) ts := httptest.NewServer(handler) defer ts.Close() - cfg.Metrics.Endpoint = ts.URL + mcfg.MetricsEndpoint = ts.URL - err := pushMetadata(cfg, mockExporterCreateSettings, &mockMetadata) + err := pushMetadata(mcfg, mockExporterCreateSettings, &mockMetadata) require.Error(t, err) } func TestPusher(t *testing.T) { - cfg := &config.Config{ - API: config.APIConfig{Key: "apikey"}, + mcfg := PusherConfig{ + APIKey: "apikey", UseResourceMetadata: true, } params := componenttest.NewNopExporterCreateSettings() @@ -210,9 +211,9 @@ func TestPusher(t *testing.T) { server := testutils.DatadogServerMock() defer server.Close() - cfg.Metrics.Endpoint = server.URL + mcfg.MetricsEndpoint = server.URL - go Pusher(ctx, params, cfg, attrs) + go Pusher(ctx, params, mcfg, attrs) body := <-server.MetadataChan var recvMetadata HostMetadata diff --git a/exporter/datadogexporter/internal/utils/http.go b/exporter/datadogexporter/internal/utils/http.go index 886f7af9bedd..ad883122ee77 100644 --- a/exporter/datadogexporter/internal/utils/http.go +++ b/exporter/datadogexporter/internal/utils/http.go @@ -23,8 +23,6 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/exporter/exporterhelper" - - "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/config" ) var ( @@ -41,7 +39,7 @@ var ( ) // NewHTTPClient returns a http.Client configured with the Agent options. -func NewHTTPClient(settings exporterhelper.TimeoutSettings, httpClientSettings config.LimitedHTTPClientSettings) *http.Client { +func NewHTTPClient(settings exporterhelper.TimeoutSettings, insecureSkipVerify bool) *http.Client { return &http.Client{ Timeout: settings.Timeout, Transport: &http.Transport{ @@ -53,7 +51,7 @@ func NewHTTPClient(settings exporterhelper.TimeoutSettings, httpClientSettings c MaxIdleConns: 100, // Not supported by intake ForceAttemptHTTP2: false, - TLSClientConfig: &tls.Config{InsecureSkipVerify: httpClientSettings.TLSSetting.InsecureSkipVerify}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}, }, } } diff --git a/exporter/datadogexporter/metrics_exporter.go b/exporter/datadogexporter/metrics_exporter.go index f23181c7bb54..b80bc7d04f7d 100644 --- a/exporter/datadogexporter/metrics_exporter.go +++ b/exporter/datadogexporter/metrics_exporter.go @@ -97,7 +97,7 @@ func translatorFromConfig(logger *zap.Logger, cfg *config.Config) (*translator.T func newMetricsExporter(ctx context.Context, params component.ExporterCreateSettings, cfg *config.Config) (*metricsExporter, error) { client := utils.CreateClient(cfg.API.Key, cfg.Metrics.TCPAddr.Endpoint) client.ExtraHeader["User-Agent"] = utils.UserAgent(params.BuildInfo) - client.HttpClient = utils.NewHTTPClient(cfg.TimeoutSettings, cfg.LimitedHTTPClientSettings) + client.HttpClient = utils.NewHTTPClient(cfg.TimeoutSettings, cfg.LimitedHTTPClientSettings.TLSSetting.InsecureSkipVerify) utils.ValidateAPIKey(params.Logger, client) @@ -163,7 +163,7 @@ func (exp *metricsExporter) PushMetricsData(ctx context.Context, md pdata.Metric if md.ResourceMetrics().Len() > 0 { attrs = md.ResourceMetrics().At(0).Resource().Attributes() } - go metadata.Pusher(exp.ctx, exp.params, exp.cfg, attrs) + go metadata.Pusher(exp.ctx, exp.params, newMetadataConfigfromConfig(exp.cfg), attrs) }) } diff --git a/exporter/datadogexporter/trace_connection.go b/exporter/datadogexporter/trace_connection.go index 0df179245493..ef86bbadd031 100644 --- a/exporter/datadogexporter/trace_connection.go +++ b/exporter/datadogexporter/trace_connection.go @@ -58,7 +58,7 @@ func createTraceEdgeConnection(rootURL, apiKey string, buildInfo component.Build statsURL: rootURL + "/api/v0.2/stats", buildInfo: buildInfo, apiKey: apiKey, - client: utils.NewHTTPClient(settings, httpClientSettings), + client: utils.NewHTTPClient(settings, httpClientSettings.TLSSetting.InsecureSkipVerify), } } diff --git a/exporter/datadogexporter/traces_exporter.go b/exporter/datadogexporter/traces_exporter.go index 07c9b3a94249..863c72903b59 100644 --- a/exporter/datadogexporter/traces_exporter.go +++ b/exporter/datadogexporter/traces_exporter.go @@ -120,7 +120,7 @@ func (exp *traceExporter) pushTraceData( if td.ResourceSpans().Len() > 0 { attrs = td.ResourceSpans().At(0).Resource().Attributes() } - go metadata.Pusher(exp.ctx, exp.params, exp.cfg, attrs) + go metadata.Pusher(exp.ctx, exp.params, newMetadataConfigfromConfig(exp.cfg), attrs) }) } From 639da2a69ed59e016448ba41a5360b72a0307469 Mon Sep 17 00:00:00 2001 From: Pablo Baeyens Date: Tue, 15 Mar 2022 15:07:47 +0100 Subject: [PATCH 4/4] `fd -e go --base-directory exporter/datadogexporter -X sd mcfg pcfg` --- .../internal/metadata/metadata.go | 30 +++++++++---------- .../internal/metadata/metadata_test.go | 24 +++++++-------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/exporter/datadogexporter/internal/metadata/metadata.go b/exporter/datadogexporter/internal/metadata/metadata.go index da43acde9ce6..0e4ed319f708 100644 --- a/exporter/datadogexporter/internal/metadata/metadata.go +++ b/exporter/datadogexporter/internal/metadata/metadata.go @@ -119,10 +119,10 @@ func metadataFromAttributes(attrs pdata.AttributeMap) *HostMetadata { return hm } -func fillHostMetadata(params component.ExporterCreateSettings, mcfg PusherConfig, hm *HostMetadata) { +func fillHostMetadata(params component.ExporterCreateSettings, pcfg PusherConfig, hm *HostMetadata) { // Could not get hostname from attributes if hm.InternalHostname == "" { - hostname := GetHost(params.Logger, mcfg.ConfigHostname) + hostname := GetHost(params.Logger, pcfg.ConfigHostname) hm.InternalHostname = hostname hm.Meta.Hostname = hostname } @@ -131,7 +131,7 @@ func fillHostMetadata(params component.ExporterCreateSettings, mcfg PusherConfig // since it does not come from OTEL conventions hm.Flavor = params.BuildInfo.Command hm.Version = params.BuildInfo.Version - hm.Tags.OTel = append(hm.Tags.OTel, mcfg.ConfigTags...) + hm.Tags.OTel = append(hm.Tags.OTel, pcfg.ConfigTags...) // EC2 data was not set from attributes if hm.Meta.EC2Hostname == "" { @@ -148,19 +148,19 @@ func fillHostMetadata(params component.ExporterCreateSettings, mcfg PusherConfig } } -func pushMetadata(mcfg PusherConfig, params component.ExporterCreateSettings, metadata *HostMetadata) error { +func pushMetadata(pcfg PusherConfig, params component.ExporterCreateSettings, metadata *HostMetadata) error { if metadata.Meta.Hostname == "" { // if the hostname is empty, don't send metadata; we don't need it. params.Logger.Debug("Skipping host metadata since the hostname is empty") return nil } - path := mcfg.MetricsEndpoint + "/intake" + path := pcfg.MetricsEndpoint + "/intake" buf, _ := json.Marshal(metadata) req, _ := http.NewRequest(http.MethodPost, path, bytes.NewBuffer(buf)) - utils.SetDDHeaders(req.Header, params.BuildInfo, mcfg.APIKey) + utils.SetDDHeaders(req.Header, params.BuildInfo, pcfg.APIKey) utils.SetExtraHeaders(req.Header, utils.JSONHeaders) - client := utils.NewHTTPClient(mcfg.TimeoutSettings, mcfg.InsecureSkipVerify) + client := utils.NewHTTPClient(pcfg.TimeoutSettings, pcfg.InsecureSkipVerify) resp, err := client.Do(req) if err != nil { @@ -180,11 +180,11 @@ func pushMetadata(mcfg PusherConfig, params component.ExporterCreateSettings, me return nil } -func pushMetadataWithRetry(retrier *utils.Retrier, params component.ExporterCreateSettings, mcfg PusherConfig, hostMetadata *HostMetadata) { +func pushMetadataWithRetry(retrier *utils.Retrier, params component.ExporterCreateSettings, pcfg PusherConfig, hostMetadata *HostMetadata) { params.Logger.Debug("Sending host metadata payload", zap.Any("payload", hostMetadata)) err := retrier.DoWithRetries(context.Background(), func(context.Context) error { - return pushMetadata(mcfg, params, hostMetadata) + return pushMetadata(pcfg, params, hostMetadata) }) if err != nil { @@ -196,12 +196,12 @@ func pushMetadataWithRetry(retrier *utils.Retrier, params component.ExporterCrea } // Pusher pushes host metadata payloads periodically to Datadog intake -func Pusher(ctx context.Context, params component.ExporterCreateSettings, mcfg PusherConfig, attrs pdata.AttributeMap) { +func Pusher(ctx context.Context, params component.ExporterCreateSettings, pcfg PusherConfig, attrs pdata.AttributeMap) { // Push metadata every 30 minutes ticker := time.NewTicker(30 * time.Minute) defer ticker.Stop() defer params.Logger.Debug("Shut down host metadata routine") - retrier := utils.NewRetrier(params.Logger, mcfg.RetrySettings, scrub.NewScrubber()) + retrier := utils.NewRetrier(params.Logger, pcfg.RetrySettings, scrub.NewScrubber()) // Get host metadata from resources and fill missing info using our exporter. // Currently we only retrieve it once but still send the same payload @@ -211,20 +211,20 @@ func Pusher(ctx context.Context, params component.ExporterCreateSettings, mcfg P // do not change over time. If this ever changes `hostMetadata` // *must* be deep copied before calling `fillHostMetadata`. hostMetadata := &HostMetadata{Meta: &Meta{}, Tags: &HostTags{}} - if mcfg.UseResourceMetadata { + if pcfg.UseResourceMetadata { hostMetadata = metadataFromAttributes(attrs) } - fillHostMetadata(params, mcfg, hostMetadata) + fillHostMetadata(params, pcfg, hostMetadata) // Run one first time at startup - pushMetadataWithRetry(retrier, params, mcfg, hostMetadata) + pushMetadataWithRetry(retrier, params, pcfg, hostMetadata) for { select { case <-ctx.Done(): return case <-ticker.C: // Send host metadata - pushMetadataWithRetry(retrier, params, mcfg, hostMetadata) + pushMetadataWithRetry(retrier, params, pcfg, hostMetadata) } } } diff --git a/exporter/datadogexporter/internal/metadata/metadata_test.go b/exporter/datadogexporter/internal/metadata/metadata_test.go index 1d618c5df5b1..22558d641cc3 100644 --- a/exporter/datadogexporter/internal/metadata/metadata_test.go +++ b/exporter/datadogexporter/internal/metadata/metadata_test.go @@ -66,13 +66,13 @@ func TestFillHostMetadata(t *testing.T) { params := componenttest.NewNopExporterCreateSettings() params.BuildInfo = mockBuildInfo - mcfg := PusherConfig{ + pcfg := PusherConfig{ ConfigHostname: "hostname", ConfigTags: []string{"key1:tag1", "key2:tag2", "env:prod"}, } metadata := &HostMetadata{Meta: &Meta{}, Tags: &HostTags{}} - fillHostMetadata(params, mcfg, metadata) + fillHostMetadata(params, pcfg, metadata) assert.Equal(t, metadata.InternalHostname, "hostname") assert.Equal(t, metadata.Flavor, "otelcontribcol") @@ -86,7 +86,7 @@ func TestFillHostMetadata(t *testing.T) { Tags: &HostTags{}, } - fillHostMetadata(params, mcfg, metadataWithVals) + fillHostMetadata(params, pcfg, metadataWithVals) assert.Equal(t, metadataWithVals.InternalHostname, "my-custom-hostname") assert.Equal(t, metadataWithVals.Flavor, "otelcontribcol") assert.Equal(t, metadataWithVals.Version, "1.0") @@ -154,7 +154,7 @@ func TestMetadataFromAttributes(t *testing.T) { } func TestPushMetadata(t *testing.T) { - mcfg := PusherConfig{ + pcfg := PusherConfig{ APIKey: "apikey", } @@ -174,14 +174,14 @@ func TestPushMetadata(t *testing.T) { ts := httptest.NewServer(handler) defer ts.Close() - mcfg.MetricsEndpoint = ts.URL + pcfg.MetricsEndpoint = ts.URL - err := pushMetadata(mcfg, mockExporterCreateSettings, &mockMetadata) + err := pushMetadata(pcfg, mockExporterCreateSettings, &mockMetadata) require.NoError(t, err) } func TestFailPushMetadata(t *testing.T) { - mcfg := PusherConfig{ + pcfg := PusherConfig{ APIKey: "apikey", } handler := http.NewServeMux() @@ -189,14 +189,14 @@ func TestFailPushMetadata(t *testing.T) { ts := httptest.NewServer(handler) defer ts.Close() - mcfg.MetricsEndpoint = ts.URL + pcfg.MetricsEndpoint = ts.URL - err := pushMetadata(mcfg, mockExporterCreateSettings, &mockMetadata) + err := pushMetadata(pcfg, mockExporterCreateSettings, &mockMetadata) require.Error(t, err) } func TestPusher(t *testing.T) { - mcfg := PusherConfig{ + pcfg := PusherConfig{ APIKey: "apikey", UseResourceMetadata: true, } @@ -211,9 +211,9 @@ func TestPusher(t *testing.T) { server := testutils.DatadogServerMock() defer server.Close() - mcfg.MetricsEndpoint = server.URL + pcfg.MetricsEndpoint = server.URL - go Pusher(ctx, params, mcfg, attrs) + go Pusher(ctx, params, pcfg, attrs) body := <-server.MetadataChan var recvMetadata HostMetadata