diff --git a/CHANGELOG.md b/CHANGELOG.md index f78b1eb..ede7a07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added: New validator `alertNameMatchesRegexp` to check if the alert name matches the regexp. - Added: New validator `groupNameMatchesRegexp` to check if the rule group name matches the regexp. - Added: New validator `recordedMetricNameMatchesRegexp` to check if the recorded metric name matches the regexp. +- Added: Set the `User-Agent` header in the Prometheus requests to `promruva` to identify the client. +- Added: Automatically configure the `X-ScopeOrgID` header in the Prometheus requests if the `source_tenants` field is set in the rule group. +- Fixed: Rules count in the result stats is now correct. +- Fixed: Better error message when validation rule is missing `scope`. - Fixed: Loading glob patterns in the file paths to rules - Fixed: Params of the `expressionCanBeEvaluated` validator were ignored, this is now fixed. - Updated: Prometheus and other dependencies diff --git a/go.mod b/go.mod index eed9119..109ed61 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/creasty/defaults v1.7.0 github.com/google/go-jsonnet v0.20.0 + github.com/grafana/dskit v0.0.0-20240528015923-27d7d41066d3 github.com/prometheus/client_golang v1.19.1 github.com/prometheus/common v0.55.0 github.com/sirupsen/logrus v1.9.3 @@ -44,7 +45,6 @@ require ( github.com/google/btree v1.1.2 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.0 // indirect - github.com/grafana/dskit v0.0.0-20240528015923-27d7d41066d3 // indirect github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 // indirect github.com/grafana/jsonparser v0.0.0-20240209175146-098958973a2d // indirect github.com/grafana/loki/pkg/push v0.0.0-20231124142027-e52380921608 // indirect diff --git a/main.go b/main.go index cac0ad1..5a1e3d2 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,9 @@ func validationRulesFromConfig(validationConfig *config.Config) ([]*validationru var validationRules []*validationrule.ValidationRule rulesIteration: for _, validationRule := range validationConfig.ValidationRules { + if validationRule.Scope == "" { + return nil, fmt.Errorf("scope is missing in the validation rule `%s`", validationRule.Name) + } for _, disabledRule := range *disabledRules { if disabledRule == validationRule.Name { continue rulesIteration diff --git a/pkg/prometheus/prometheus.go b/pkg/prometheus/prometheus.go index 7ddec15..f19c220 100644 --- a/pkg/prometheus/prometheus.go +++ b/pkg/prometheus/prometheus.go @@ -8,9 +8,11 @@ import ( "os" "path" "strings" + "sync" "time" "github.com/fusakla/promruval/v2/pkg/config" + "github.com/grafana/dskit/user" "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" prom_config "github.com/prometheus/common/config" @@ -20,6 +22,7 @@ import ( const ( bearerTokenEnvVar = "PROMETHEUS_BEARER_TOKEN" + userAgent = "promruval" ) func loadBearerToken(promConfig config.PrometheusConfig) (string, error) { @@ -55,32 +58,51 @@ func NewClient(promConfig config.PrometheusConfig) (*Client, error) { } func NewClientWithRoundTripper(promConfig config.PrometheusConfig, tripper http.RoundTripper) (*Client, error) { + headers := prom_config.Headers{ + Headers: map[string]prom_config.Header{ + "User-Agent": {Values: []string{userAgent}}, + }, + } cli, err := api.NewClient(api.Config{ Address: promConfig.URL, - RoundTripper: tripper, + RoundTripper: prom_config.NewHeadersRoundTripper(&headers, tripper), }) if err != nil { return nil, fmt.Errorf("failed to initialize prometheus client: %w", err) } v1cli := v1.NewAPI(cli) promClient := Client{ - apiClient: v1cli, - url: promConfig.URL, - timeout: promConfig.Timeout, - queryOffset: promConfig.QueryOffset, - queryLookback: promConfig.QueryLookback, - cache: newCache(promConfig.CacheFile, promConfig.URL, promConfig.MaxCacheAge), + apiClient: v1cli, + httpHeaders: &headers, + httpHeadersMtx: sync.Mutex{}, + url: promConfig.URL, + timeout: promConfig.Timeout, + queryOffset: promConfig.QueryOffset, + queryLookback: promConfig.QueryLookback, + cache: newCache(promConfig.CacheFile, promConfig.URL, promConfig.MaxCacheAge), } return &promClient, nil } type Client struct { - apiClient v1.API - url string - timeout time.Duration - queryOffset time.Duration - queryLookback time.Duration - cache *cache + apiClient v1.API + httpHeaders *prom_config.Headers + httpHeadersMtx sync.Mutex + url string + timeout time.Duration + queryOffset time.Duration + queryLookback time.Duration + cache *cache +} + +func (s *Client) SetSourceTenants(sourceTenants []string) { + s.httpHeadersMtx.Lock() + s.httpHeaders.Headers[user.OrgIDHeaderName] = prom_config.Header{Values: sourceTenants} +} + +func (s *Client) ClearSourceTenants() { + delete(s.httpHeaders.Headers, user.OrgIDHeaderName) + s.httpHeadersMtx.Unlock() } func (s *Client) queryTimeRange() (start, end time.Time) { @@ -103,9 +125,11 @@ func (s *Client) newContext() (context.Context, context.CancelFunc) { return ctx, cancel } -func (s *Client) SelectorMatch(selector string) ([]model.LabelSet, error) { +func (s *Client) SelectorMatch(selector string, sourceTenants []string) ([]model.LabelSet, error) { ctx, cancel := s.newContext() defer cancel() + s.SetSourceTenants(sourceTenants) + defer s.ClearSourceTenants() start := time.Now() queryStart, queryEnd := s.queryTimeRange() result, warnings, err := s.apiClient.Series(ctx, []string{selector}, queryStart, queryEnd) @@ -119,11 +143,11 @@ func (s *Client) SelectorMatch(selector string) ([]model.LabelSet, error) { return result, nil } -func (s *Client) SelectorMatchingSeries(selector string) (int, error) { +func (s *Client) SelectorMatchingSeries(selector string, sourceTenants []string) (int, error) { if count, found := s.cache.SelectorMatchingSeries[selector]; found { return count, nil } - series, err := s.SelectorMatch(selector) + series, err := s.SelectorMatch(selector, sourceTenants) if err != nil { return 0, err } @@ -131,10 +155,12 @@ func (s *Client) SelectorMatchingSeries(selector string) (int, error) { return len(series), nil } -func (s *Client) Labels() ([]string, error) { +func (s *Client) Labels(sourceTenants []string) ([]string, error) { if len(s.cache.KnownLabels) == 0 { ctx, cancel := s.newContext() defer cancel() + s.SetSourceTenants(sourceTenants) + defer s.ClearSourceTenants() start := time.Now() queryStart, queryEnd := s.queryTimeRange() result, warnings, err := s.apiClient.LabelNames(ctx, []string{}, queryStart, queryEnd) @@ -150,14 +176,15 @@ func (s *Client) Labels() ([]string, error) { return s.cache.KnownLabels, nil } -func (s *Client) Query(query string) ([]*model.Sample, int, time.Duration, error) { - var duration time.Duration +func (s *Client) Query(query string, sourceTenants []string) ([]*model.Sample, int, time.Duration, error) { ctx, cancel := s.newContext() defer cancel() + s.SetSourceTenants(sourceTenants) + defer s.ClearSourceTenants() start := time.Now() _, queryEnd := s.queryTimeRange() result, warnings, err := s.apiClient.Query(ctx, query, queryEnd) - duration = time.Since(start) + duration := time.Since(start) log.Debugf("query `%s` on %s prometheus took %s", query, s.url, duration) if err != nil { return nil, 0, 0, fmt.Errorf("error querying prometheus: %w", err) @@ -182,11 +209,11 @@ func (s *Client) Query(query string) ([]*model.Sample, int, time.Duration, error return nil, 0, 0, fmt.Errorf("unknown prometheus response type: %s", result) } -func (s *Client) QueryStats(query string) (int, time.Duration, error) { +func (s *Client) QueryStats(query string, sourceTenants []string) (int, time.Duration, error) { if stats, found := s.cache.QueriesStats[query]; found { return stats.Series, stats.Duration, stats.Error } - _, series, duration, err := s.Query(query) + _, series, duration, err := s.Query(query, sourceTenants) stats := queryStats{Series: series, Duration: duration, Error: err} s.cache.QueriesStats[query] = stats return stats.Series, stats.Duration, stats.Error diff --git a/pkg/validate/validate.go b/pkg/validate/validate.go index 271d1a0..443722b 100644 --- a/pkg/validate/validate.go +++ b/pkg/validate/validate.go @@ -111,6 +111,7 @@ func Files(fileNames []string, validationRules []*validationrule.ValidationRule, groupReport.Valid = false } for _, ruleNode := range group.Rules { + validationReport.RulesCount++ originalRule := ruleNode.OriginalRule() var ruleReport *report.RuleReport if originalRule.Alert != "" { diff --git a/pkg/validator/promql_expression.go b/pkg/validator/promql_expression.go index 6a3b669..4094a0d 100644 --- a/pkg/validator/promql_expression.go +++ b/pkg/validator/promql_expression.go @@ -308,13 +308,13 @@ func (h expressionCanBeEvaluated) String() string { return msg } -func (h expressionCanBeEvaluated) Validate(_ unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error { +func (h expressionCanBeEvaluated) Validate(group unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error { var errs []error if prometheusClient == nil { log.Error("missing the `prometheus` section of configuration for querying prometheus, skipping check that requires it...") return nil } - count, duration, err := prometheusClient.QueryStats(rule.Expr) + count, duration, err := prometheusClient.QueryStats(rule.Expr, group.SourceTenants) if err != nil { return append(errs, err) } @@ -341,7 +341,7 @@ func (h expressionUsesExistingLabels) String() string { return "expression uses only labels that are actually present in Prometheus" } -func (h expressionUsesExistingLabels) Validate(_ unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error { +func (h expressionUsesExistingLabels) Validate(group unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error { if prometheusClient == nil { log.Error("missing the `prometheus` section of configuration for querying prometheus, skipping check that requires it...") return nil @@ -351,7 +351,7 @@ func (h expressionUsesExistingLabels) Validate(_ unmarshaler.RuleGroup, rule rul return []error{err} } var errs []error - knownLabels, err := prometheusClient.Labels() + knownLabels, err := prometheusClient.Labels(group.SourceTenants) if err != nil { return []error{err} } @@ -390,7 +390,7 @@ func (h expressionSelectorsMatchesAnything) String() string { return "expression selectors actually matches any series in Prometheus" } -func (h expressionSelectorsMatchesAnything) Validate(_ unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error { +func (h expressionSelectorsMatchesAnything) Validate(group unmarshaler.RuleGroup, rule rulefmt.Rule, prometheusClient *prometheus.Client) []error { if prometheusClient == nil { log.Error("missing the `prometheus` section of configuration for querying prometheus, skipping check that requires it...") return nil @@ -401,7 +401,7 @@ func (h expressionSelectorsMatchesAnything) Validate(_ unmarshaler.RuleGroup, ru return []error{err} } for _, s := range selectors { - matchingSeries, err := prometheusClient.SelectorMatchingSeries(s) + matchingSeries, err := prometheusClient.SelectorMatchingSeries(s, group.SourceTenants) if err != nil { errs = append(errs, err) continue