From 9c2064abb1b790c08826ead27dc2587c6cb75d7d Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 11 Feb 2020 17:06:21 +0100 Subject: [PATCH] Fix: don't miss address scheme (#16205) * Fix: don't miss address scheme * Add unit test * Adjust source after code review * Add comment to method --- CHANGELOG.next.asciidoc | 1 + metricbeat/mb/lightmetricset.go | 18 +++- metricbeat/mb/lightmodules_test.go | 91 +++++++++++++++++++ .../httpextended/extends/manifest.yml | 4 + .../lightmodules/httpextended/module.yml | 3 + 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 metricbeat/mb/testdata/lightmodules/httpextended/extends/manifest.yml create mode 100644 metricbeat/mb/testdata/lightmodules/httpextended/module.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a4034b65e6e..989c46e557a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -87,6 +87,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Change lookup_fields from metricset.host to service.address {pull}15883[15883] - Add dedot for cloudwatch metric name. {issue}15916[15916] {pull}15917[15917] - Fixed issue `logstash-xpack` module suddenly ceasing to monitor Logstash. {issue}15974[15974] {pull}16044[16044] +- Fix skipping protocol scheme by light modules. {pull}16205[pull] *Packetbeat* diff --git a/metricbeat/mb/lightmetricset.go b/metricbeat/mb/lightmetricset.go index dee1d0afb7b..11e49cd5c77 100644 --- a/metricbeat/mb/lightmetricset.go +++ b/metricbeat/mb/lightmetricset.go @@ -18,6 +18,9 @@ package mb import ( + "fmt" + "net/url" + "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" @@ -83,7 +86,8 @@ func (m *LightMetricSet) Registration(r *Register) (MetricSetRegistration, error // At this point host parser was already run, we need to run this again // with the overriden defaults if registration.HostParser != nil { - base.hostData, err = registration.HostParser(base.module, base.host) + host := m.useHostURISchemeIfPossible(base.host, base.hostData.URI) + base.hostData, err = registration.HostParser(base.module, host) if err != nil { return nil, errors.Wrapf(err, "host parser failed on light metricset factory for '%s/%s'", m.Module, m.Name) } @@ -96,6 +100,18 @@ func (m *LightMetricSet) Registration(r *Register) (MetricSetRegistration, error return registration, nil } +// useHostURISchemeIfPossible method parses given URI to extract protocol scheme and prepend it to the host. +// It prevents from skipping protocol scheme (e.g. https) while executing HostParser. +func (m *LightMetricSet) useHostURISchemeIfPossible(host, uri string) string { + u, err := url.ParseRequestURI(uri) + if err == nil { + if u.Scheme != "" { + return fmt.Sprintf("%s://%s", u.Scheme, u.Host) + } + } + return host +} + // baseModule does the configuration overrides in the base module configuration // taking into account the light metric set default configurations func (m *LightMetricSet) baseModule(from Module) (*BaseModule, error) { diff --git a/metricbeat/mb/lightmodules_test.go b/metricbeat/mb/lightmodules_test.go index e7a933cc47a..90f81242651 100644 --- a/metricbeat/mb/lightmodules_test.go +++ b/metricbeat/mb/lightmodules_test.go @@ -20,6 +20,7 @@ package mb import ( + "net/url" "testing" "time" @@ -287,6 +288,96 @@ func TestNewModuleFromConfig(t *testing.T) { } } +func TestLightMetricSet_VerifyHostDataURI(t *testing.T) { + const hostEndpoint = "ceph-restful:8003" + const sampleHttpsEndpoint = "https://" + hostEndpoint + + r := NewRegister() + r.MustAddMetricSet("http", "json", newMetricSetWithOption, + WithHostParser(func(module Module, host string) (HostData, error) { + u, err := url.Parse(host) + if err != nil { + return HostData{}, err + } + return HostData{ + Host: u.Host, + URI: host, + }, nil + })) + r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) + + config, err := common.NewConfigFrom( + common.MapStr{ + "module": "httpextended", + "metricsets": []string{"extends"}, + "hosts": []string{sampleHttpsEndpoint}, + }) + require.NoError(t, err) + + _, metricSets, err := NewModule(config, r) + require.NoError(t, err) + require.Len(t, metricSets, 1) + + assert.Equal(t, hostEndpoint, metricSets[0].Host()) + assert.Equal(t, sampleHttpsEndpoint, metricSets[0].HostData().URI) +} + +func TestLightMetricSet_WithoutHostParser(t *testing.T) { + const sampleHttpsEndpoint = "https://ceph-restful:8003" + + r := NewRegister() + r.MustAddMetricSet("http", "json", newMetricSetWithOption) + r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) + + config, err := common.NewConfigFrom( + common.MapStr{ + "module": "httpextended", + "metricsets": []string{"extends"}, + "hosts": []string{sampleHttpsEndpoint}, + }) + require.NoError(t, err) + + _, metricSets, err := NewModule(config, r) + require.NoError(t, err) + require.Len(t, metricSets, 1) + + assert.Equal(t, sampleHttpsEndpoint, metricSets[0].Host()) + assert.Equal(t, sampleHttpsEndpoint, metricSets[0].HostData().URI) +} + +func TestLightMetricSet_VerifyHostDataURI_NonParsableHost(t *testing.T) { + const ( + postgresHost = "host1:5432" + postgresEndpoint = "postgres://user1:pass@host1:5432?connect_timeout=2" + postgresParsed = "connect_timeout=3 host=host1 password=pass port=5432 user=user1" + ) + + r := NewRegister() + r.MustAddMetricSet("http", "json", newMetricSetWithOption, + WithHostParser(func(module Module, host string) (HostData, error) { + return HostData{ + Host: postgresHost, + URI: postgresParsed, + }, nil + })) + r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) + + config, err := common.NewConfigFrom( + common.MapStr{ + "module": "httpextended", + "metricsets": []string{"extends"}, + "hosts": []string{postgresEndpoint}, + }) + require.NoError(t, err) + + _, metricSets, err := NewModule(config, r) + require.NoError(t, err) + require.Len(t, metricSets, 1) + + assert.Equal(t, postgresHost, metricSets[0].Host()) + assert.Equal(t, postgresParsed, metricSets[0].HostData().URI) +} + func TestNewModulesCallModuleFactory(t *testing.T) { logp.TestingSetup() diff --git a/metricbeat/mb/testdata/lightmodules/httpextended/extends/manifest.yml b/metricbeat/mb/testdata/lightmodules/httpextended/extends/manifest.yml new file mode 100644 index 00000000000..a56f13a3c7d --- /dev/null +++ b/metricbeat/mb/testdata/lightmodules/httpextended/extends/manifest.yml @@ -0,0 +1,4 @@ +default: true +input: + module: http + metricset: json diff --git a/metricbeat/mb/testdata/lightmodules/httpextended/module.yml b/metricbeat/mb/testdata/lightmodules/httpextended/module.yml new file mode 100644 index 00000000000..93ecaa59ba1 --- /dev/null +++ b/metricbeat/mb/testdata/lightmodules/httpextended/module.yml @@ -0,0 +1,3 @@ +name: httpextended +metricsets: +- extends