Set accept header for prometheus client scraping (#17291) (#17325)

* Set accept header for prometheus to match the same as prometheus upstream.

* Fix go.mod.

* Add changelog entry.

(cherry picked from commit 268511d)

CHANGELOG.next.asciidoc

@@ -176,6 +176,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Check if CCR feature is available on Elasticsearch cluster before attempting to call CCR APIs from `elasticsearch/ccr` metricset. {issue}16511[16511] {pull}17073[17073]
 - Use max in k8s overview dashboard aggregations. {pull}17015[17015]
 - Fix Disk Used and Disk Usage visualizations in the Metricbeat System dashboards. {issue}12435[12435] {pull}17272[17272]
+- Fix missing Accept header for Prometheus and OpenMetrics module. {issue}16870[16870] {pull}17291[17291]
 
 *Packetbeat*
 

metricbeat/helper/http.go

@@ -39,7 +39,7 @@ import (
 type HTTP struct {
 	hostData mb.HostData
 	client   *http.Client // HTTP client that is reused across requests.
-	headers  map[string]string
+	headers  http.Header
 	name     string
 	uri      string
 	method   string
@@ -58,16 +58,20 @@ func NewHTTP(base mb.BaseMetricSet) (*HTTP, error) {
 
 // newHTTPWithConfig creates a new http helper from some configuration
 func newHTTPFromConfig(config Config, name string, hostData mb.HostData) (*HTTP, error) {
+	headers := http.Header{}
 	if config.Headers == nil {
 		config.Headers = map[string]string{}
 	}
+	for k, v := range config.Headers {
+		headers.Set(k, v)
+	}
 
 	if config.BearerTokenFile != "" {
 		header, err := getAuthHeaderFromToken(config.BearerTokenFile)
 		if err != nil {
 			return nil, err
 		}
-		config.Headers["Authorization"] = header
+		headers.Set("Authorization", header)
 	}
 
 	tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS)
@@ -103,7 +107,7 @@ func newHTTPFromConfig(config Config, name string, hostData mb.HostData) (*HTTP,
 			},
 			Timeout: config.Timeout,
 		},
-		headers: config.Headers,
+		headers: headers,
 		method:  "GET",
 		uri:     hostData.SanitizedURI,
 		body:    nil,
@@ -124,14 +128,11 @@ func (h *HTTP) FetchResponse() (*http.Response, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create HTTP request")
 	}
+	req.Header = h.headers
 	if h.hostData.User != "" || h.hostData.Password != "" {
 		req.SetBasicAuth(h.hostData.User, h.hostData.Password)
 	}
 
-	for k, v := range h.headers {
-		req.Header.Set(k, v)
-	}
-
 	resp, err := h.client.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("error making http request: %v", err)
@@ -142,7 +143,17 @@ func (h *HTTP) FetchResponse() (*http.Response, error) {
 
 // SetHeader sets HTTP headers to use in requests
 func (h *HTTP) SetHeader(key, value string) {
-	h.headers[key] = value
+	h.headers.Set(key, value)
+}
+
+// SetHeaderDefault sets HTTP header as default
+//
+// Note: This will only set the header when the header is not already set.
+func (h *HTTP) SetHeaderDefault(key, value string) {
+	c := h.headers.Get(key)
+	if c == "" {
+		h.headers.Set(key, value)
+	}
 }
 
 // SetMethod sets HTTP method to use in requests

metricbeat/helper/http_test.go

@@ -162,6 +162,34 @@ func TestAuthentication(t *testing.T) {
 	assert.Equal(t, http.StatusOK, response.StatusCode, "response status code")
 }
 
+func TestSetHeader(t *testing.T) {
+	cfg := defaultConfig()
+	cfg.Headers = map[string]string{
+		"Override": "default",
+	}
+
+	h, err := newHTTPFromConfig(cfg, "test", mb.HostData{})
+	require.NoError(t, err)
+
+	h.SetHeader("Override", "overridden")
+	v := h.headers.Get("override")
+	assert.Equal(t, "overridden", v)
+}
+
+func TestSetHeaderDefault(t *testing.T) {
+	cfg := defaultConfig()
+	cfg.Headers = map[string]string{
+		"Override": "default",
+	}
+
+	h, err := newHTTPFromConfig(cfg, "test", mb.HostData{})
+	require.NoError(t, err)
+
+	h.SetHeaderDefault("Override", "overridden")
+	v := h.headers.Get("override")
+	assert.Equal(t, "default", v)
+}
+
 func TestOverUnixSocket(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skipf("unix domain socket aren't supported under Windows")

metricbeat/helper/prometheus/prometheus.go

@@ -33,6 +33,8 @@ import (
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
+const acceptHeader = `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1`
+
 // Prometheus helper retrieves prometheus formatted metrics
 type Prometheus interface {
 	// GetFamilies requests metric families from prometheus endpoint and returns them
@@ -55,10 +57,11 @@ type httpfetcher interface {
 // NewPrometheusClient creates new prometheus helper
 func NewPrometheusClient(base mb.BaseMetricSet) (Prometheus, error) {
 	http, err := helper.NewHTTP(base)
-
 	if err != nil {
 		return nil, err
 	}
+
+	http.SetHeaderDefault("Accept", acceptHeader)
 	return &prometheus{http, base.Logger()}, nil
 }