feat(backend): add headers option to history

Fixes #5422.

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## v0.116
+
+### Added
+
+- Added `headers` option to `history:rewrite` config block. This allows to set
+  custom headers passed to Prometheus when sending history query requests.
+
 ## v0.115
 
 ### Fixed

cmd/karma/alert_history.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/prymitive/karma/internal/alertmanager"
 	"github.com/prymitive/karma/internal/config"
+	"github.com/prymitive/karma/internal/mapper"
 	"github.com/prymitive/karma/internal/slices"
 	uriUtil "github.com/prymitive/karma/internal/uri"
 )
@@ -157,8 +158,8 @@ func (hp *historyPoller) run(workers int) {
 }
 
 func (hp *historyPoller) stop() {
-	hp.isRunning.Store(false)
 	log.Debug().Msg("Stopping history poller")
+	hp.isRunning.Store(false)
 	close(hp.queue)
 }
 
@@ -193,7 +194,7 @@ func (hp *historyPoller) knownBadLookup(key string) (*knownBadUpstream, bool) {
 func (hp *historyPoller) startWorker(wid int) {
 	log.Debug().Int("worker", wid).Int("queue", cap(hp.queue)).Dur("timeout", hp.queryTimeout).Msg("Starting history poller")
 	for j := range hp.queue {
-		sourceURI := rewriteSource(config.Config.History.Rewrite, j.uri)
+		sourceURI, headers := rewriteSource(config.Config.History.Rewrite, j.uri)
 		expiredAt := time.Now().Add(time.Minute * -5)
 		key := hashQuery(sourceURI, j.labels)
 		if kb, found := hp.knownBadLookup(key); found && kb.timestamp.After(expiredAt) {
@@ -224,6 +225,9 @@ func (hp *historyPoller) startWorker(wid int) {
 				Err(err).
 				Msg("Error while configuring HTTP transport for history request")
 		}
+		if len(headers) > 0 {
+			transport = mapper.SetHeaders(transport, headers)
+		}
 		values, err := countAlerts(sourceURI, hp.queryTimeout, transport, j.labels)
 		if err != nil {
 			log.Error().
@@ -252,7 +256,7 @@ func hashQuery(uri string, labels map[string]string) string {
 	return fmt.Sprintf("%x", hasher.Sum(nil))
 }
 
-func rewriteSource(rules []config.HistoryRewrite, uri string) string {
+func rewriteSource(rules []config.HistoryRewrite, uri string) (string, map[string]string) {
 	// trim trailing / to ensure all URIs are without a /
 	uri = strings.TrimSuffix(uri, "/")
 	for _, rule := range rules {
@@ -264,9 +268,9 @@ func rewriteSource(rules []config.HistoryRewrite, uri string) string {
 			result = rule.SourceRegex.ExpandString(result, rule.URI, uri, submatches)
 		}
 		log.Debug().Str("source", uri).Str("uri", string(result)).Msg("Alert history source rewrite")
-		return string(result)
+		return string(result), rule.Headers
 	}
-	return uri
+	return uri, nil
 }
 
 func rewriteTransport(rules []config.HistoryRewrite, uri string) (http.RoundTripper, error) {

cmd/karma/alert_history_test.go

@@ -90,6 +90,7 @@ func TestAlertHistory(t *testing.T) {
 	type mock struct {
 		method    string
 		uri       *regexp.Regexp
+		matcher   httpmock.Matcher
 		responder httpmock.Responder
 	}
 
@@ -531,6 +532,60 @@ func TestAlertHistory(t *testing.T) {
 				},
 			},
 		},
+		{
+			mocks: []mock{
+				{
+					method:  "POST",
+					uri:     regexp.MustCompile("^http://localhost:9101/api/v1/labels"),
+					matcher: httpmock.HeaderIs("X-Auth", "secret").WithName("X-Auth"),
+					responder: httpmock.NewJsonResponderOrPanic(200, prometheusAPIV1Labels{
+						Status: "success",
+						Data:   []string{"alertname", "instance", "job"},
+					}),
+				},
+				{
+					method:  "POST",
+					uri:     regexp.MustCompile("^http://localhost:9101/api/v1/query_range"),
+					matcher: httpmock.HeaderIs("X-Auth", "secret").WithName("X-Auth"),
+					responder: httpmock.NewJsonResponderOrPanic(200, prometheusAPIV1QueryRange{
+						Status: "success",
+						Data: generateV1Matrix(
+							[]seriesValues{
+								{
+									metric: model.Metric{
+										"alertname": "Fake Alert",
+									},
+									values: generateIntSlice(0, 1, 24),
+								},
+							}, time.Hour),
+					}),
+				},
+			},
+			config: cfg{
+				enabled: true,
+				timeout: time.Second * 5,
+				workers: 5,
+				rewrite: []config.HistoryRewrite{
+					{
+						SourceRegex: regex.MustCompileAnchored("(.+)"),
+						URI:         "$1",
+						Headers:     map[string]string{"X-Auth": "secret"},
+					},
+				},
+			},
+			queries: []historyQuery{
+				{
+					payload: generateHistoryPayload(AlertHistoryPayload{
+						Sources: []string{"http://localhost:9101"},
+						Labels:  map[string]string{"alertname": "Fake Alert", "cluster": "prod"},
+					}),
+					code: 200,
+					response: AlertHistoryResponse{
+						Samples: generateHistorySamples(generateIntSlice(0, 1, 24), time.Hour),
+					},
+				},
+			},
+		},
 	}
 
 	defer func() {
@@ -540,9 +595,6 @@ func TestAlertHistory(t *testing.T) {
 		config.Config.History.Rewrite = []config.HistoryRewrite{}
 	}()
 
-	httpmock.Activate()
-	defer httpmock.DeactivateAndReset()
-
 	mockConfig(t.Setenv)
 
 	hp := newHistoryPoller(1, time.Second*5)
@@ -558,10 +610,11 @@ func TestAlertHistory(t *testing.T) {
 	zerolog.SetGlobalLevel(zerolog.FatalLevel)
 	for i, tc := range testCases {
 		t.Run(fmt.Sprintf("%d/enabled=%v", i, tc.config.enabled), func(t *testing.T) {
-			httpmock.Reset()
+			httpmock.Activate()
+			defer httpmock.DeactivateAndReset()
 
 			for _, mock := range tc.mocks {
-				httpmock.RegisterRegexpResponder(mock.method, mock.uri, mock.responder)
+				httpmock.RegisterRegexpMatcherResponder(mock.method, mock.uri, mock.matcher, mock.responder)
 				t.Logf("Registered responder %s %s", mock.method, mock.uri)
 			}
 			config.Config.History.Enabled = tc.config.enabled
@@ -684,7 +737,7 @@ func TestRewriteSource(t *testing.T) {
 
 	for i, tc := range testCases {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			uri := rewriteSource(tc.rules, tc.uri)
+			uri, _ := rewriteSource(tc.rules, tc.uri)
 			if diff := cmp.Diff(tc.out, uri); diff != "" {
 				t.Errorf("Incorrect rewriteSource result (-want +got):\n%s", diff)
 			}

cmd/karma/tests/testscript/060_log_full_config_file.txt

@@ -184,6 +184,7 @@ level=info msg="        cert: \"\""
 level=info msg="        key: \"\""
 level=info msg="        insecureSkipVerify: false"
 level=info msg="      proxy_url: \"\""
+level=info msg="      headers: {}"
 level=info msg="    - source: (.+)"
 level=info msg="      uri: $1"
 level=info msg="      tls:"
@@ -192,6 +193,7 @@ level=info msg="        cert: /etc/server.pem"
 level=info msg="        key: /etc/server.key"
 level=info msg="        insecureSkipVerify: true"
 level=info msg="      proxy_url: \"\""
+level=info msg="      headers: {}"
 level=info msg="karma:"
 level=info msg="  name: karma-demo"
 level=info msg="labels:"

docs/CONFIGURATION.md

@@ -268,6 +268,10 @@ alertmanager:
 - `headers` - a map with a list of key: values which are header: value.
   These custom headers will be sent with every request to the alert manager
   instance.
+  **NOTE**: these headers are only sent for alertmanager requests, they are NOT set
+  on requests send to Prometheus server when querying alert history.
+  Please see `history:rewrite` section below if you want to set headers
+  for Prometheus requests.
 - `cors:credentials` - sets the
   [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) credentials
   settings for browser requests,
@@ -760,6 +764,8 @@ history:
     - source: regex
       uri: string
       proxy_url: string
+      headers:
+        any: string
       tls:
         ca: string
         cert: string
@@ -842,6 +848,18 @@ history:
       proxy_url: socks5://proxy.local:5000
 ```
 
+Example with rewrite rule that will set an extra header for all history request
+send to Prometheus server `http://prometheus.example.com`:
+
+```YAML
+history:
+  rewrite:
+    - source: 'http://prometheus.example.com'
+      headers:
+        X-Auth: secret
+        X-Foo: bar
+```
+
 ### Karma
 
 `karma` section allows configuring miscellaneous internal options.

internal/config/models.go

@@ -61,11 +61,12 @@ type AuthorizationGroup struct {
 }
 
 type HistoryRewrite struct {
-	Source      string          `yaml:"source"`
-	SourceRegex *regexp.Regexp  `yaml:"-"`
-	URI         string          `yaml:"uri"`
-	TLS         AlertmanagerTLS `yaml:"tls" koanf:"tls"`
-	ProxyURL    string          `yaml:"proxy_url" koanf:"proxy_url"`
+	Source      string            `yaml:"source"`
+	SourceRegex *regexp.Regexp    `yaml:"-"`
+	URI         string            `yaml:"uri"`
+	TLS         AlertmanagerTLS   `yaml:"tls" koanf:"tls"`
+	ProxyURL    string            `yaml:"proxy_url" koanf:"proxy_url"`
+	Headers     map[string]string `yaml:"headers" koanf:"headers"`
 }
 
 type configSchema struct {