Request prometheus endpoints to be gzipped by default (#20766)

* Request prometheus endpoints to be gzipped by default

CHANGELOG.next.asciidoc

@@ -659,6 +659,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547]
 - Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641]
 - Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391]
+- Request prometheus endpoints to be gzipped by default {pull}20766[20766]
 
 *Packetbeat*
 

metricbeat/helper/prometheus/prometheus.go

@@ -18,6 +18,7 @@
 package prometheus
 
 import (
+	"compress/gzip"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -63,19 +64,33 @@ func NewPrometheusClient(base mb.BaseMetricSet) (Prometheus, error) {
 	}
 
 	http.SetHeaderDefault("Accept", acceptHeader)
+	http.SetHeaderDefault("Accept-Encoding", "gzip")
 	return &prometheus{http, base.Logger()}, nil
 }
 
 // GetFamilies requests metric families from prometheus endpoint and returns them
 func (p *prometheus) GetFamilies() ([]*dto.MetricFamily, error) {
+	var reader io.Reader
+
 	resp, err := p.FetchResponse()
 	if err != nil {
 		return nil, err
 	}
 	defer resp.Body.Close()
 
+	if resp.Header.Get("Content-Encoding") == "gzip" {
+		greader, err := gzip.NewReader(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		defer greader.Close()
+		reader = greader
+	} else {
+		reader = resp.Body
+	}
+
 	if resp.StatusCode > 399 {
-		bodyBytes, err := ioutil.ReadAll(resp.Body)
+		bodyBytes, err := ioutil.ReadAll(reader)
 		if err == nil {
 			p.logger.Debug("error received from prometheus endpoint: ", string(bodyBytes))
 		}
@@ -87,7 +102,7 @@ func (p *prometheus) GetFamilies() ([]*dto.MetricFamily, error) {
 		return nil, fmt.Errorf("Invalid format for response of response")
 	}
 
-	decoder := expfmt.NewDecoder(resp.Body, format)
+	decoder := expfmt.NewDecoder(reader, format)
 	if decoder == nil {
 		return nil, fmt.Errorf("Unable to create decoder to decode response")
 	}

metricbeat/helper/prometheus/prometheus_test.go

@@ -19,6 +19,7 @@ package prometheus
 
 import (
 	"bytes"
+	"compress/gzip"
 	"io/ioutil"
 	"net/http"
 	"sort"
@@ -185,10 +186,17 @@ var _ = httpfetcher(&mockFetcher{})
 // FetchResponse returns an HTTP response but for the Body, which
 // returns the mockFetcher.Response contents
 func (m mockFetcher) FetchResponse() (*http.Response, error) {
+	body := bytes.NewBuffer(nil)
+	writer := gzip.NewWriter(body)
+	writer.Write([]byte(m.response))
+	writer.Close()
+
 	return &http.Response{
 		StatusCode: 200,
-		Header:     make(http.Header),
-		Body:       ioutil.NopCloser(bytes.NewReader([]byte(m.response))),
+		Header: http.Header{
+			"Content-Encoding": []string{"gzip"},
+		},
+		Body: ioutil.NopCloser(body),
 	}, nil
 }