Fix: don't miss address scheme (elastic#16205)

* Fix: don't miss address scheme

* Add unit test

* Adjust source after code review

* Add comment to method

CHANGELOG.next.asciidoc

@@ -87,6 +87,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change lookup_fields from metricset.host to service.address {pull}15883[15883]
 - Add dedot for cloudwatch metric name. {issue}15916[15916] {pull}15917[15917]
 - Fixed issue `logstash-xpack` module suddenly ceasing to monitor Logstash. {issue}15974[15974] {pull}16044[16044]
+- Fix skipping protocol scheme by light modules. {pull}16205[pull]
 
 *Packetbeat*
 

metricbeat/mb/lightmetricset.go

@@ -18,6 +18,9 @@
 package mb
 
 import (
+ "fmt"
+ "net/url"
+
  "github.com/pkg/errors"
 
  "github.com/elastic/beats/libbeat/common"
@@ -83,7 +86,8 @@ func (m *LightMetricSet) Registration(r *Register) (MetricSetRegistration, error
  // At this point host parser was already run, we need to run this again
  // with the overriden defaults
  if registration.HostParser != nil {
- base.hostData, err = registration.HostParser(base.module, base.host)
+ host := m.useHostURISchemeIfPossible(base.host, base.hostData.URI)
+ base.hostData, err = registration.HostParser(base.module, host)
  if err != nil {
  return nil, errors.Wrapf(err, "host parser failed on light metricset factory for '%s/%s'", m.Module, m.Name)
  }
@@ -96,6 +100,18 @@ func (m *LightMetricSet) Registration(r *Register) (MetricSetRegistration, error
  return registration, nil
 }
 
+// useHostURISchemeIfPossible method parses given URI to extract protocol scheme and prepend it to the host.
+// It prevents from skipping protocol scheme (e.g. https) while executing HostParser.
+func (m *LightMetricSet) useHostURISchemeIfPossible(host, uri string) string {
+ u, err := url.ParseRequestURI(uri)
+ if err == nil {
+ if u.Scheme != "" {
+ return fmt.Sprintf("%s://%s", u.Scheme, u.Host)
+ }
+ }
+ return host
+}
+
 // baseModule does the configuration overrides in the base module configuration
 // taking into account the light metric set default configurations
 func (m *LightMetricSet) baseModule(from Module) (*BaseModule, error) {

metricbeat/mb/lightmodules_test.go

@@ -20,6 +20,7 @@
 package mb
 
 import (
+ "net/url"
  "testing"
  "time"
 
@@ -287,6 +288,96 @@ func TestNewModuleFromConfig(t *testing.T) {
  }
 }
 
+func TestLightMetricSet_VerifyHostDataURI(t *testing.T) {
+ const hostEndpoint = "ceph-restful:8003"
+ const sampleHttpsEndpoint = "https://" + hostEndpoint
+
+ r := NewRegister()
+ r.MustAddMetricSet("http", "json", newMetricSetWithOption,
+ WithHostParser(func(module Module, host string) (HostData, error) {
+ u, err := url.Parse(host)
+ if err != nil {
+ return HostData{}, err
+ }
+ return HostData{
+ Host: u.Host,
+ URI: host,
+ }, nil
+ }))
+ r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules"))
+
+ config, err := common.NewConfigFrom(
+ common.MapStr{
+ "module": "httpextended",
+ "metricsets": []string{"extends"},
+ "hosts": []string{sampleHttpsEndpoint},
+ })
+ require.NoError(t, err)
+
+ _, metricSets, err := NewModule(config, r)
+ require.NoError(t, err)
+ require.Len(t, metricSets, 1)
+
+ assert.Equal(t, hostEndpoint, metricSets[0].Host())
+ assert.Equal(t, sampleHttpsEndpoint, metricSets[0].HostData().URI)
+}
+
+func TestLightMetricSet_WithoutHostParser(t *testing.T) {
+ const sampleHttpsEndpoint = "https://ceph-restful:8003"
+
+ r := NewRegister()
+ r.MustAddMetricSet("http", "json", newMetricSetWithOption)
+ r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules"))
+
+ config, err := common.NewConfigFrom(
+ common.MapStr{
+ "module": "httpextended",
+ "metricsets": []string{"extends"},
+ "hosts": []string{sampleHttpsEndpoint},
+ })
+ require.NoError(t, err)
+
+ _, metricSets, err := NewModule(config, r)
+ require.NoError(t, err)
+ require.Len(t, metricSets, 1)
+
+ assert.Equal(t, sampleHttpsEndpoint, metricSets[0].Host())
+ assert.Equal(t, sampleHttpsEndpoint, metricSets[0].HostData().URI)
+}
+
+func TestLightMetricSet_VerifyHostDataURI_NonParsableHost(t *testing.T) {
+ const (
+ postgresHost = "host1:5432"
+ postgresEndpoint = "postgres://user1:pass@host1:5432?connect_timeout=2"
+ postgresParsed = "connect_timeout=3 host=host1 password=pass port=5432 user=user1"
+ )
+
+ r := NewRegister()
+ r.MustAddMetricSet("http", "json", newMetricSetWithOption,
+ WithHostParser(func(module Module, host string) (HostData, error) {
+ return HostData{
+ Host: postgresHost,
+ URI: postgresParsed,
+ }, nil
+ }))
+ r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules"))
+
+ config, err := common.NewConfigFrom(
+ common.MapStr{
+ "module": "httpextended",
+ "metricsets": []string{"extends"},
+ "hosts": []string{postgresEndpoint},
+ })
+ require.NoError(t, err)
+
+ _, metricSets, err := NewModule(config, r)
+ require.NoError(t, err)
+ require.Len(t, metricSets, 1)
+
+ assert.Equal(t, postgresHost, metricSets[0].Host())
+ assert.Equal(t, postgresParsed, metricSets[0].HostData().URI)
+}
+
 func TestNewModulesCallModuleFactory(t *testing.T) {
  logp.TestingSetup()
 

metricbeat/mb/testdata/lightmodules/httpextended/extends/manifest.yml

@@ -0,0 +1,4 @@
+default: true
+input:
+ module: http
+ metricset: json

metricbeat/mb/testdata/lightmodules/httpextended/module.yml

@@ -0,0 +1,3 @@
+name: httpextended
+metricsets:
+- extends