feat: add hasSourceTenantsForMetrics validator (#37)

* feat: add hasSourceTenantsForMetrics validator, refactor, optimize docs html rendering, add some additional tests

Signed-off-by: Martin Chodur <m.chodur@seznam.cz>

* feat: additional things

Signed-off-by: Martin Chodur <m.chodur@seznam.cz>

* refactor: expression helpers

Signed-off-by: Martin Chodur <m.chodur@seznam.cz>

* chore: changelog

Signed-off-by: Martin Chodur <m.chodur@seznam.cz>

---------

Signed-off-by: Martin Chodur <m.chodur@seznam.cz>

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 - Added new validator `expressionDoesNotUseMetrics`, see its [docs](docs/validations.md#expressiondoesnotusemetrics).
+- Added new validator `hasSourceTenantsForMetrics`, see its [docs](docs/validations.md#hassourcetenantsformetrics).
+- Improved the HTML output of human readable validation description.
+- Added examples of the human-readable validation descriptions to the examples dir.
+- Refactored the validation so it can use also group to validate the context of the rule.
 
 ## [v2.6.0] - 2023-12-06
 - Added new validator `expressionWithNoMetricName`, see its [docs](docs/validations.md#expressionwithnometricname). Thanks @tizki !

Makefile

@@ -9,8 +9,10 @@ PROMRUVAL_BIN := ./promruval
 E2E_TESTS_VALIDATIONS_FILE := examples/validation.yaml
 E2E_TESTS_ADDITIONAL_VALIDATIONS_FILE := examples/additional-validation.yaml
 E2E_TESTS_RULES_FILES := examples/rules/*.yaml
+E2E_TESTS_DOCS_FILE_MD := examples/human_readable.md
+E2E_TESTS_DOCS_FILE_HTML := examples/human_readable.html
 
-all: deps lint build test e2e-test
+all: clean deps lint build test e2e-test
 
 $(TMP_DIR):
 	mkdir -p $(TMP_DIR)
@@ -36,7 +38,8 @@ build:
 
 e2e-test: build
 	$(PROMRUVAL_BIN) validate --config-file $(E2E_TESTS_VALIDATIONS_FILE) --config-file $(E2E_TESTS_ADDITIONAL_VALIDATIONS_FILE) $(E2E_TESTS_RULES_FILES)
-	$(PROMRUVAL_BIN) validation-docs --config-file $(E2E_TESTS_VALIDATIONS_FILE) --config-file $(E2E_TESTS_ADDITIONAL_VALIDATIONS_FILE)
+	$(PROMRUVAL_BIN) validation-docs --config-file $(E2E_TESTS_VALIDATIONS_FILE) --config-file $(E2E_TESTS_ADDITIONAL_VALIDATIONS_FILE) > $(E2E_TESTS_DOCS_FILE_MD)
+	$(PROMRUVAL_BIN) validation-docs --config-file $(E2E_TESTS_VALIDATIONS_FILE) --config-file $(E2E_TESTS_ADDITIONAL_VALIDATIONS_FILE) --output=html > $(E2E_TESTS_DOCS_FILE_HTML)
 
 docker: build
 	docker build -t fusakla/promruval .

README.md

@@ -36,7 +36,8 @@ or [watch a lightning talk about it from PromCon](https://www.youtube.com/watch?
 - and many more...
 
 Validations are quite variable, so you can use them as you fit.
-Full list of supported validations can be found [here](docs/validations.md).
+
+**👉 Full list of supported validations can be found [here](docs/validations.md).**
 
 In case of any missing, please create a feature request!
 
@@ -65,7 +66,7 @@ Prometheus rules validation tool.
 
 Flags:
       --help   Show context-sensitive help (also try --help-long and --help-man).
-  -c, --config-file=CONFIG-FILE ...  
+  -c, --config-file=CONFIG-FILE ...
                Path to validation config file. Can be passed multiple times, only validationRules will be reflected from the additional configs.
       --debug  Enable debug logging.
 
@@ -81,17 +82,17 @@ Commands:
   validate [<flags>] <path>...
     Validate Prometheus rule files using validation rules from config file.
 
-    -d, --disable-rule=DISABLE-RULE ...  
+    -d, --disable-rule=DISABLE-RULE ...
                                    Allows to disable any validation rules by it's name. Can be passed multiple times.
-    -e, --enable-rule=ENABLE-RULE ...  
+    -e, --enable-rule=ENABLE-RULE ...
                                    Only enable these validation rules. Can be passed multiple times.
     -o, --output=[text,json,yaml]  Format of the output.
         --color                    Use color output.
 
   validation-docs [<flags>]
     Print human readable form of the validation rules from config file.
 
-    -o, --output=[text,markdown,html]  
+    -o, --output=[text,markdown,html]
       Format of the output.
 ```
 
@@ -132,7 +133,7 @@ customExcludeAnnotation: my_disable_annotation
 prometheus:
   # URL of the running prometheus instance to be used
   url: https://foo.bar/
-  # OPTIONAL Skip TLS verification 
+  # OPTIONAL Skip TLS verification
   insecureSkipTlsVerify: false
   # OPTIONAL Timeout for any request on the Prometheus instance
   timeout: 30s
@@ -243,7 +244,8 @@ groups:
 If you want more human readable validation summary (for a documentation or generating readable git pages)
 you can use the `validation-docs` command, see the [usage](#usage).
 It should print out more human readable form than the configuration file is
-and supports multiple output formats.
+and supports multiple output formats such as `text`, `markdown` and `HTML`.
+See the examples for the output for [Markdown](./examples/human_readable.md) and [HTML](./examples/human_readable.html).
 
 ```bash
 promruval validation-docs --config-file examples/validation.yaml --output=html

docs/validations.md

@@ -32,6 +32,8 @@
     - [`expressionWithNoMetricName`](#expressionwithnometricname)
   - [Alert](#alert)
     - [`forIsNotLongerThan`](#forisnotlongerthan)
+  - [Others](#others)
+    - [`hasSourceTenantsForMetrics`](#hassourcetenantsformetrics)
 
 ## Labels
 
@@ -288,3 +290,16 @@ Fails if the alert uses longer `for` than the specified limit.
 params:
   limit: "1h"
 ```
+## Others
+
+### `hasSourceTenantsForMetrics`
+
+Fails, if the rule uses metric, that matches the specified regular expression for any tenant, but does not have the tenant configured in the  `source_tenants` of the rule group option the rule belongs to.
+
+```yaml
+params:
+  sourceTenants:
+    <tenant_name>: <metric_name_regexp> # The regexp will be fully anchored (surrounded by ^...$)
+    # Example:
+    # k8s: "kube_.*|container_.*"
+```

examples/human_readable.html

@@ -0,0 +1,52 @@
+
+<h1>Validation rules</h1>
+
+  <h2><a href="#check-severity-label">check-severity-label</a></h2>
+	  <ul>
+		<li>Alert has labels: <code>severity</code></li>
+		<li>Alert label <code>severity</code> has one of the allowed values: <code>info</code>,<code>warning</code>,<code>critical</code></li>
+		<li>Alert if rule has label <code>severity</code> with value <code>info</code> , it cannot have label <code>page</code></li>
+		<li>Alert expression can be successfully evaluated on the live Prometheus instance</li>
+		<li>Alert expression uses only labels that are actually present in Prometheus</li>
+		<li>Alert expression selectors actually matches any series in Prometheus</li>
+		<li>Alert expression does not use data older than <code>6h0m0s</code></li>
+	  </ul>
+
+  <h2><a href="#check-team-label">check-team-label</a></h2>
+	  <ul>
+		<li>Alert has labels: <code>xxx</code></li>
+		<li>Alert label <code>team</code> has one of the allowed values: <code>sre@company.com</code></li>
+	  </ul>
+
+  <h2><a href="#check-playbook-annotation">check-playbook-annotation</a></h2>
+	  <ul>
+		<li>Alert has any of these annotations: <code>playbook</code>,<code>link</code></li>
+		<li>Alert Annotation <code>link</code> is a valid URL and does not return HTTP status 404</li>
+	  </ul>
+
+  <h2><a href="#check-alert-title">check-alert-title</a></h2>
+	  <ul>
+		<li>Alert has all of these annotations: <code>title</code></li>
+	  </ul>
+
+  <h2><a href="#check-prometheus-limitations">check-prometheus-limitations</a></h2>
+	  <ul>
+		<li>All rules expression does not use data older than <code>6h0m0s</code></li>
+		<li>All rules does not use any of the <code>cluster</code>,<code>locality</code>,<code>prometheus-type</code>,<code>replica</code> labels is in its expression</li>
+	  </ul>
+
+  <h2><a href="#check-source-tenants">check-source-tenants</a></h2>
+	  <ul>
+		<li>All rules verifies if the rule group, the rule belongs to, has the required source_tenants configured, according to the mapping of metric names to tenants: <code>k8s</code>:<code>^container_.*|kube_.*$</code></li>
+	  </ul>
+
+  <h2><a href="#check-metric-name">check-metric-name</a></h2>
+	  <ul>
+		<li>Alert expression with no metric name</li>
+	  </ul>
+
+  <h2><a href="#another-checks">another-checks</a></h2>
+	  <ul>
+		<li>All rules labels does not have empty values</li>
+	  </ul>
+

examples/human_readable.md

@@ -0,0 +1,36 @@
+
+Validation rules:
+
+  check-severity-label
+    - Alert has labels: `severity`
+    - Alert label `severity` has one of the allowed values: `info`,`warning`,`critical`
+    - Alert if rule has label `severity` with value `info` , it cannot have label `page`
+    - Alert expression can be successfully evaluated on the live Prometheus instance
+    - Alert expression uses only labels that are actually present in Prometheus
+    - Alert expression selectors actually matches any series in Prometheus
+    - Alert expression does not use data older than `6h0m0s`
+
+  check-team-label
+    - Alert has labels: `xxx`
+    - Alert label `team` has one of the allowed values: `sre@company.com`
+
+  check-playbook-annotation
+    - Alert has any of these annotations: `playbook`,`link`
+    - Alert Annotation `link` is a valid URL and does not return HTTP status 404
+
+  check-alert-title
+    - Alert has all of these annotations: `title`
+
+  check-prometheus-limitations
+    - All rules expression does not use data older than `6h0m0s`
+    - All rules does not use any of the `cluster`,`locality`,`prometheus-type`,`replica` labels is in its expression
+
+  check-source-tenants
+    - All rules verifies if the rule group, the rule belongs to, has the required source_tenants configured, according to the mapping of metric names to tenants: `k8s`:`^container_.*|kube_.*$`
+
+  check-metric-name
+    - Alert expression with no metric name
+
+  another-checks
+    - All rules labels does not have empty values
+

examples/rules/rules.yaml

@@ -29,13 +29,14 @@ groups:
           disabled_validation_rules: check-team-label,check-prometheus-limitations
 
   - name: testIgnoreValidationsInExpr
+    source_tenants: ["k8s"]
     rules:
       - alert: test
         expr: |
           # Comment before.
           # Comment on the same line. ignore_validations: labelHasAllowedValue
           # Comment after.
-          foo{
+          kube_pod_labels{
             # ignore_validations: expressionSelectorsMatchesAnything, hasLabels
           }
         for: 1m

examples/validation.yaml

@@ -10,11 +10,11 @@ validationRules:
     validations:
       - type: hasLabels
         params:
-          labels: [ "severity" ]
+          labels: ["severity"]
       - type: labelHasAllowedValue
         params:
           label: "severity"
-          allowedValues: [ "info", "warning", "critical" ]
+          allowedValues: ["info", "warning", "critical"]
       - type: exclusiveLabels
         params:
           firstLabel: severity
@@ -35,7 +35,7 @@ validationRules:
     validations:
       - type: hasLabels
         params:
-          labels: [ "xxx" ]
+          labels: ["xxx"]
       - type: labelHasAllowedValue
         params:
           label: "team"
@@ -47,7 +47,7 @@ validationRules:
     validations:
       - type: hasAnyOfAnnotations
         params:
-          annotations: [ "playbook", "link" ]
+          annotations: ["playbook", "link"]
       - type: annotationIsValidURL
         params:
           annotation: "link"
@@ -58,7 +58,7 @@ validationRules:
     validations:
       - type: hasAnnotations
         params:
-          annotations: [ "title" ]
+          annotations: ["title"]
 
   - name: check-prometheus-limitations
     scope: All rules
@@ -68,7 +68,15 @@ validationRules:
           limit: "6h"
       - type: expressionDoesNotUseLabels
         params:
-          labels: [ "cluster", "locality", "prometheus-type", "replica" ]
+          labels: ["cluster", "locality", "prometheus-type", "replica"]
+
+  - name: check-source-tenants
+    scope: All rules
+    validations:
+      - type: hasSourceTenantsForMetrics
+        params:
+          sourceTenants:
+            k8s: "container_.*|kube_.*"
 
   - name: check-metric-name
     scope: Alert

go.mod

@@ -9,6 +9,8 @@ require (
 	github.com/prometheus/client_golang v1.17.0
 	github.com/prometheus/common v0.45.0
 	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.8.4
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools v2.2.0+incompatible
@@ -55,14 +57,12 @@ require (
 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/prometheus/prometheus v0.48.0
-	github.com/stretchr/testify v1.8.4 // indirect
 	go.opentelemetry.io/otel v1.21.0 // indirect
 	go.opentelemetry.io/otel/metric v1.21.0 // indirect
 	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/crypto v0.15.0 // indirect
-	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect

main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/fusakla/promruval/v2/pkg/prometheus"
 	"github.com/fusakla/promruval/v2/pkg/report"
 	"github.com/fusakla/promruval/v2/pkg/validate"
+	"github.com/fusakla/promruval/v2/pkg/validationrule"
 	"github.com/fusakla/promruval/v2/pkg/validator"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"
@@ -55,22 +56,22 @@ func loadConfigFile(configFilePath string) (*config.Config, error) {
 	return validationConfig, nil
 }
 
-func validationRulesFromConfig(config *config.Config) ([]*validate.ValidationRule, error) {
-	var validationRules []*validate.ValidationRule
+func validationRulesFromConfig(config *config.Config) ([]*validationrule.ValidationRule, error) {
+	var validationRules []*validationrule.ValidationRule
 rulesIteration:
-	for _, rule := range config.ValidationRules {
+	for _, validationRule := range config.ValidationRules {
 		for _, disabledRule := range *disabledRules {
-			if disabledRule == rule.Name {
+			if disabledRule == validationRule.Name {
 				continue rulesIteration
 			}
 		}
 		for _, enabledRule := range *enabledRules {
-			if enabledRule != rule.Name {
+			if enabledRule != validationRule.Name {
 				continue rulesIteration
 			}
 		}
-		newRule := validate.NewValidationRule(rule.Name, rule.Scope)
-		for _, validatorConfig := range rule.Validations {
+		newRule := validationrule.New(validationRule.Name, validationRule.Scope)
+		for _, validatorConfig := range validationRule.Validations {
 			newValidator, err := validator.NewFromConfig(validatorConfig)
 			if err != nil {
 				return nil, fmt.Errorf("loading validator config: %w", err)

pkg/config/config.go

@@ -2,9 +2,10 @@ package config
 
 import (
 	"fmt"
-	"gopkg.in/yaml.v3"
 	"time"
 
+	"gopkg.in/yaml.v3"
+
 	"github.com/creasty/defaults"
 )
 

pkg/report/report.go

@@ -12,7 +12,7 @@ import (
 
 type ValidationRule interface {
 	Name() string
-	Scope() string
+	Scope() config.ValidationScope
 	ValidationTexts() []string
 }