Merge branch 'master' into fus-remote-mite

CHANGELOG.md

@@ -30,6 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added: New validator `alertNameMatchesRegexp` to check if the alert name matches the regexp.
 - Added: New validator `groupNameMatchesRegexp` to check if the rule group name matches the regexp.
 - Added: New validator `recordedMetricNameMatchesRegexp` to check if the recorded metric name matches the regexp.
+- Added: Set the `User-Agent` header in the Prometheus requests to `promruva` to identify the client.
+- Added: Automatically configure the `X-ScopeOrgID` header in the Prometheus requests if the `source_tenants` field is set in the rule group.
+- Fixed: Rules count in the result stats is now correct.
+- Fixed: Better error message when validation rule is missing `scope`.
 - Fixed: Loading glob patterns in the file paths to rules
 - Fixed: Params of the `expressionCanBeEvaluated` validator were ignored, this is now fixed.
 - Updated: Prometheus and other dependencies

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/bmatcuk/doublestar/v4 v4.6.1
 	github.com/creasty/defaults v1.7.0
 	github.com/google/go-jsonnet v0.20.0
+	github.com/grafana/dskit v0.0.0-20240528015923-27d7d41066d3
 	github.com/prometheus/client_golang v1.19.1
 	github.com/prometheus/common v0.55.0
 	github.com/sirupsen/logrus v1.9.3
@@ -44,7 +45,6 @@ require (
 	github.com/google/btree v1.1.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/grafana/dskit v0.0.0-20240528015923-27d7d41066d3 // indirect
 	github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 // indirect
 	github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d // indirect
 	github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 // indirect

main.go

@@ -55,6 +55,9 @@ func validationRulesFromConfig(validationConfig *config.Config) ([]*validationru
 	var validationRules []*validationrule.ValidationRule
 rulesIteration:
 	for _, validationRule := range validationConfig.ValidationRules {
+		if validationRule.Scope == "" {
+			return nil, fmt.Errorf("scope is missing in the validation rule `%s`", validationRule.Name)
+		}
 		for _, disabledRule := range *disabledRules {
 			if disabledRule == validationRule.Name {
 				continue rulesIteration

pkg/prometheus/prometheus.go

@@ -8,9 +8,11 @@ import (
 	"os"
 	"path"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/fusakla/promruval/v2/pkg/config"
+	"github.com/grafana/dskit/user"
 	"github.com/prometheus/client_golang/api"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	prom_config "github.com/prometheus/common/config"
@@ -20,6 +22,7 @@ import (
 
 const (
 	bearerTokenEnvVar = "PROMETHEUS_BEARER_TOKEN"
+	userAgent         = "promruval"
 )
 
 func loadBearerToken(promConfig config.PrometheusConfig) (string, error) {
@@ -55,32 +58,51 @@ func NewClient(promConfig config.PrometheusConfig) (*Client, error) {
 }
 
 func NewClientWithRoundTripper(promConfig config.PrometheusConfig, tripper http.RoundTripper) (*Client, error) {
+	headers := prom_config.Headers{
+		Headers: map[string]prom_config.Header{
+			"User-Agent": {Values: []string{userAgent}},
+		},
+	}
 	cli, err := api.NewClient(api.Config{
 		Address:      promConfig.URL,
-		RoundTripper: tripper,
+		RoundTripper: prom_config.NewHeadersRoundTripper(&headers, tripper),
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize prometheus client: %w", err)
 	}
 	v1cli := v1.NewAPI(cli)
 	promClient := Client{
-		apiClient:     v1cli,
-		url:           promConfig.URL,
-		timeout:       promConfig.Timeout,
-		queryOffset:   promConfig.QueryOffset,
-		queryLookback: promConfig.QueryLookback,
-		cache:         newCache(promConfig.CacheFile, promConfig.URL, promConfig.MaxCacheAge),
+		apiClient:      v1cli,
+		httpHeaders:    &headers,
+		httpHeadersMtx: sync.Mutex{},
+		url:            promConfig.URL,
+		timeout:        promConfig.Timeout,
+		queryOffset:    promConfig.QueryOffset,
+		queryLookback:  promConfig.QueryLookback,
+		cache:          newCache(promConfig.CacheFile, promConfig.URL, promConfig.MaxCacheAge),
 	}
 	return &promClient, nil
 }
 
 type Client struct {
-	apiClient     v1.API
-	url           string
-	timeout       time.Duration
-	queryOffset   time.Duration
-	queryLookback time.Duration
-	cache         *cache
+	apiClient      v1.API
+	httpHeaders    *prom_config.Headers
+	httpHeadersMtx sync.Mutex
+	url            string
+	timeout        time.Duration
+	queryOffset    time.Duration
+	queryLookback  time.Duration
+	cache          *cache
+}
+
+func (s *Client) SetSourceTenants(sourceTenants []string) {
+	s.httpHeadersMtx.Lock()
+	s.httpHeaders.Headers[user.OrgIDHeaderName] = prom_config.Header{Values: sourceTenants}
+}
+
+func (s *Client) ClearSourceTenants() {
+	delete(s.httpHeaders.Headers, user.OrgIDHeaderName)
+	s.httpHeadersMtx.Unlock()
 }
 
 func (s *Client) queryTimeRange() (start, end time.Time) {
@@ -103,9 +125,11 @@ func (s *Client) newContext() (context.Context, context.CancelFunc) {
 	return ctx, cancel
 }
 
-func (s *Client) SelectorMatch(selector string) ([]model.LabelSet, error) {
+func (s *Client) SelectorMatch(selector string, sourceTenants []string) ([]model.LabelSet, error) {
 	ctx, cancel := s.newContext()
 	defer cancel()
+	s.SetSourceTenants(sourceTenants)
+	defer s.ClearSourceTenants()
 	start := time.Now()
 	queryStart, queryEnd := s.queryTimeRange()
 	result, warnings, err := s.apiClient.Series(ctx, []string{selector}, queryStart, queryEnd)
@@ -119,22 +143,24 @@ func (s *Client) SelectorMatch(selector string) ([]model.LabelSet, error) {
 	return result, nil
 }
 
-func (s *Client) SelectorMatchingSeries(selector string) (int, error) {
+func (s *Client) SelectorMatchingSeries(selector string, sourceTenants []string) (int, error) {
 	if count, found := s.cache.SelectorMatchingSeries[selector]; found {
 		return count, nil
 	}
-	series, err := s.SelectorMatch(selector)
+	series, err := s.SelectorMatch(selector, sourceTenants)
 	if err != nil {
 		return 0, err
 	}
 	s.cache.SelectorMatchingSeries[selector] = len(series)
 	return len(series), nil
 }
 
-func (s *Client) Labels() ([]string, error) {
+func (s *Client) Labels(sourceTenants []string) ([]string, error) {
 	if len(s.cache.KnownLabels) == 0 {
 		ctx, cancel := s.newContext()
 		defer cancel()
+		s.SetSourceTenants(sourceTenants)
+		defer s.ClearSourceTenants()
 		start := time.Now()
 		queryStart, queryEnd := s.queryTimeRange()
 		result, warnings, err := s.apiClient.LabelNames(ctx, []string{}, queryStart, queryEnd)
@@ -150,14 +176,15 @@ func (s *Client) Labels() ([]string, error) {
 	return s.cache.KnownLabels, nil
 }
 
-func (s *Client) Query(query string) ([]*model.Sample, int, time.Duration, error) {
-	var duration time.Duration
+func (s *Client) Query(query string, sourceTenants []string) ([]*model.Sample, int, time.Duration, error) {
 	ctx, cancel := s.newContext()
 	defer cancel()
+	s.SetSourceTenants(sourceTenants)
+	defer s.ClearSourceTenants()
 	start := time.Now()
 	_, queryEnd := s.queryTimeRange()
 	result, warnings, err := s.apiClient.Query(ctx, query, queryEnd)
-	duration = time.Since(start)
+	duration := time.Since(start)
 	log.Debugf("query `%s` on %s prometheus took %s", query, s.url, duration)
 	if err != nil {
 		return nil, 0, 0, fmt.Errorf("error querying prometheus: %w", err)
@@ -182,11 +209,11 @@ func (s *Client) Query(query string) ([]*model.Sample, int, time.Duration, error
 	return nil, 0, 0, fmt.Errorf("unknown prometheus response type: %s", result)
 }
 
-func (s *Client) QueryStats(query string) (int, time.Duration, error) {
+func (s *Client) QueryStats(query string, sourceTenants []string) (int, time.Duration, error) {
 	if stats, found := s.cache.QueriesStats[query]; found {
 		return stats.Series, stats.Duration, stats.Error
 	}
-	_, series, duration, err := s.Query(query)
+	_, series, duration, err := s.Query(query, sourceTenants)
 	stats := queryStats{Series: series, Duration: duration, Error: err}
 	s.cache.QueriesStats[query] = stats
 	return stats.Series, stats.Duration, stats.Error

pkg/validate/validate.go

@@ -111,6 +111,7 @@ func Files(fileNames []string, validationRules []*validationrule.ValidationRule,
 				groupReport.Valid = false
 			}
 			for _, ruleNode := range group.Rules {
+				validationReport.RulesCount++
 				originalRule := ruleNode.OriginalRule()
 				var ruleReport *report.RuleReport
 				if originalRule.Alert != "" {

pkg/validator/promql_expression.go

@@ -308,13 +308,13 @@ func (h expressionCanBeEvaluated) String() string {
 	return msg
 }
 
-func (h expressionCanBeEvaluated) Validate(_ unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error {
+func (h expressionCanBeEvaluated) Validate(group unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error {
 	var errs []error
 	if prometheusClient == nil {
 		log.Error("missing the `prometheus` section of configuration for querying prometheus, skipping check that requires it...")
 		return nil
 	}
-	count, duration, err := prometheusClient.QueryStats(rule.Expr)
+	count, duration, err := prometheusClient.QueryStats(rule.Expr, group.SourceTenants)
 	if err != nil {
 		return append(errs, err)
 	}
@@ -341,7 +341,7 @@ func (h expressionUsesExistingLabels) String() string {
 	return "expression uses only labels that are actually present in Prometheus"
 }
 
-func (h expressionUsesExistingLabels) Validate(_ unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error {
+func (h expressionUsesExistingLabels) Validate(group unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error {
 	if prometheusClient == nil {
 		log.Error("missing the `prometheus` section of configuration for querying prometheus, skipping check that requires it...")
 		return nil
@@ -351,7 +351,7 @@ func (h expressionUsesExistingLabels) Validate(_ unmarshaler.RuleGroup, rule rul
 		return []error{err}
 	}
 	var errs []error
-	knownLabels, err := prometheusClient.Labels()
+	knownLabels, err := prometheusClient.Labels(group.SourceTenants)
 	if err != nil {
 		return []error{err}
 	}
@@ -390,7 +390,7 @@ func (h expressionSelectorsMatchesAnything) String() string {
 	return "expression selectors actually matches any series in Prometheus"
 }
 
-func (h expressionSelectorsMatchesAnything) Validate(_ unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error {
+func (h expressionSelectorsMatchesAnything) Validate(group unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error {
 	if prometheusClient == nil {
 		log.Error("missing the `prometheus` section of configuration for querying prometheus, skipping check that requires it...")
 		return nil
@@ -401,7 +401,7 @@ func (h expressionSelectorsMatchesAnything) Validate(_ unmarshaler.RuleGroup, ru
 		return []error{err}
 	}
 	for _, s := range selectors {
-		matchingSeries, err := prometheusClient.SelectorMatchingSeries(s)
+		matchingSeries, err := prometheusClient.SelectorMatchingSeries(s, group.SourceTenants)
 		if err != nil {
 			errs = append(errs, err)
 			continue