Merge pull request cloudflare#291 from cloudflare/settings

Add extra settings for promql/series

cmd/pint/scan.go

@@ -49,6 +49,12 @@ func checkRules(ctx context.Context, workers int, cfg config.Config, entries []d
 	results := make(chan reporter.Report, workers*5)
 	wg := sync.WaitGroup{}
 
+	for _, s := range cfg.Check {
+		settings, _ := s.Decode()
+		key := checks.SettingsKey(s.Name)
+		ctx = context.WithValue(ctx, key, settings)
+	}
+
 	for w := 1; w <= workers; w++ {
 		wg.Add(1)
 		go func() {

cmd/pint/tests/0025_config.txt

@@ -36,8 +36,21 @@ level=info msg="Loading configuration file" path=.pint.hcl
       "promql/series",
       "rule/label",
       "rule/reject"
+    ],
+    "disabled": [
+      "promql/fragile"
     ]
   },
+  "check": [
+    {
+      "ignoreMetrics": [
+        ".*_error",
+        ".*_error_.*",
+        ".*_errors",
+        ".*_errors_.*"
+      ]
+    }
+  ],
   "rules": [
     {},
     {
@@ -111,9 +124,6 @@ level=info msg="Loading configuration file" path=.pint.hcl
         "bytesPerSample": 4036
       }
     }
-  ],
-  "PrometheusServers": [
-    {}
   ]
 }
 -- .pint.hcl --
@@ -123,6 +133,19 @@ prometheus "prom" {
   required = true
 }
 
+checks {
+  disabled = ["promql/fragile"]
+}
+
+check "promql/series" {
+  ignoreMetrics = [
+    ".*_error",
+    ".*_error_.*",
+    ".*_errors",
+    ".*_errors_.*",
+  ]
+}
+
 rule{  }
 
 rule {

cmd/pint/tests/0079_check_promql_series_invalid.txt

@@ -0,0 +1,22 @@
+pint.error --no-color config
+! stdout .
+cmp stderr stderr.txt
+
+-- stderr.txt --
+level=info msg="Loading configuration file" path=.pint.hcl
+level=fatal msg="Fatal error" error="failed to load config file \".pint.hcl\": .pint.hcl:8,3-6: Unsupported argument; An argument named \"bob\" is not expected here."
+-- .pint.hcl --
+prometheus "prom" {
+  uri     = "https://"
+  timeout = "2m"
+  required = true
+}
+
+check "promql/series" {
+  bob = [
+    ".*_error",
+    ".*_error_.*",
+    ".*_errors",
+    ".*_errors_.*",
+  ]
+}

cmd/pint/tests/0080_lint_online.txt

@@ -0,0 +1,122 @@
+exec bash -x ./prometheus.sh &
+exec bash -c 'I=0 ; while [ ! -f prometheus.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done'
+
+pint.ok --no-color lint rules
+! stdout .
+cmp stderr stderr.txt
+exec bash -c 'cat prometheus.pid | xargs kill'
+
+-- stderr.txt --
+level=info msg="Loading configuration file" path=.pint.hcl
+level=info msg="File parsed" path=rules/1.yml rules=1
+rules/1.yml:2: prometheus "prom1" at http://127.0.0.1:7080 didn't have any series for "http_errors_total" metric in the last 1w. Metric name "http_errors_total" matches "promql/series" check ignore regexp "^.+_errors_.+$" (promql/series)
+  expr: rate(http_errors_total[2m]) > 0
+
+level=info msg="Problems found" Warning=1
+-- rules/1.yml --
+- alert: http errors
+  expr: rate(http_errors_total[2m]) > 0
+
+-- .pint.hcl --
+prometheus "prom1" {
+  uri     = "http://127.0.0.1:7080"
+  timeout = "5s"
+  required = true
+}
+parser {
+  relaxed = [".*"]
+}
+check "promql/series" {
+    ignoreMetrics = [
+        ".+_error",
+        ".+_error_.+",
+        ".+_errors",
+        ".+_errors_.+",
+    ]
+}
+
+-- prometheus.go --
+package main
+
+import (
+	"context"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+)
+
+func main() {
+	http.HandleFunc("/api/v1/metadata", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"status":"success","data":{}}`))
+	})
+
+	http.HandleFunc("/api/v1/status/config", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"status":"success","data":{"yaml":"global:\n  scrape_interval: 30s\n"}}`))
+	})
+
+	http.HandleFunc("/api/v1/query_range", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{
+            "status":"success",
+            "data":{
+                "resultType":"matrix",
+                "result":[]
+            }
+        }`))
+	})
+
+	http.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{
+            "status":"success",
+            "data":{
+                "resultType":"vector",
+                "result":[]
+            }
+        }`))
+	})
+
+	listener, err := net.Listen("tcp", "127.0.0.1:7080")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	server := &http.Server{
+		Addr: "127.0.0.1:7080",
+	}
+
+	go func() {
+		_ = server.Serve(listener)
+	}()
+
+	pid := os.Getpid()
+	err = os.WriteFile("prometheus.pid", []byte(strconv.Itoa(pid)), 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		time.Sleep(time.Minute * 2)
+		stop <- syscall.SIGTERM
+	}()
+	<-stop
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	server.Shutdown(ctx)
+}
+
+-- prometheus.sh --
+env GOCACHE=$TMPDIR go run prometheus.go

docs/changelog.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## v0.23.0
+
+### Added
+
+- Added extra global configuration for `promql/series` check.
+  See check [documentation](checks/promql/series.md) for details.
+
 ## v0.22.2
 
 ### Fixed

docs/checks/promql/series.md

@@ -121,8 +121,32 @@ that.
 
 ## Configuration
 
-This check doesn't have any configuration options but it supports a few control
-comments that can be placed around each rule.
+This check supports setting extra configuration option to fine tune its behavior.
+
+Syntax:
+
+```js
+check "promql/series" {
+  ignoreMetrics = [ "(.*)", ... ]
+}
+```
+
+- `ignoreMetrics` - list of regexp matchers, if a metric is missing from Prometheus
+  but the name matches any of provided regexp matchers then pint will only report a
+  warning, instead of a bug level report.
+
+Example:
+
+```js
+check "promql/series" {
+  ignoreMetrics = [
+    ".*_error",
+    ".*_error_.*",
+    ".*_errors",
+    ".*_errors_.*",
+  ]
+}
+```
 
 ### min-age
 

docs/examples/ignore_error_metrics.hcl

@@ -0,0 +1,23 @@
+# Define "prod" Prometheus instance that will only be used for
+# rules defined in file matching "alerting/prod/.+" or "recording/prod/.+".
+prometheus "prod" {
+  uri     = "https://prod.example.com"
+  timeout = "30s"
+
+  paths = [
+    "alerting/prod/.+",
+    "recording/prod/.+",
+  ]
+}
+
+# Extra global configuration for the promql/series check.
+check "promql/series" {
+  # Don't report missing metrics for any metric with name matching
+  # one of the regexp matchers below.
+  ignoreMetrics = [
+    ".+_error",
+    ".+_error_.+",
+    ".+_errors",
+    ".+_errors_.+",
+  ]
+}

internal/checks/base.go

@@ -83,6 +83,8 @@ const (
 	Fatal
 )
 
+type SettingsKey string
+
 type Problem struct {
 	Fragment string
 	Lines    []int

internal/checks/base_test.go

@@ -82,10 +82,13 @@ type problemsFn func(string) []checks.Problem
 
 type newPrometheusFn func(string) *promapi.FailoverGroup
 
+type newCtxFn func() context.Context
+
 type checkTest struct {
 	description string
 	content     string
 	prometheus  newPrometheusFn
+	ctx         newCtxFn
 	checker     newCheckFn
 	entries     []discovery.Entry
 	problems    problemsFn
@@ -121,7 +124,11 @@ func runTests(t *testing.T, testCases []checkTest) {
 			entries, err := parseContent(tc.content)
 			require.NoError(t, err, "cannot parse rule content")
 			for _, entry := range entries {
-				problems := tc.checker(prom).Check(context.Background(), entry.Rule, tc.entries)
+				ctx := context.Background()
+				if tc.ctx != nil {
+					ctx = tc.ctx()
+				}
+				problems := tc.checker(prom).Check(ctx, entry.Rule, tc.entries)
 				require.Equal(t, tc.problems(uri), problems)
 			}