Add watch tests

cmd/pint/tests/0032_ci_github.txt

@@ -1,5 +1,5 @@
 exec bash -x ./webserver.sh &
-#exec bash -c 'while [ ! -f server.pid ]; do sleep 1 ; done'
+exec bash -c 'while [ ! -f server.pid ]; do sleep 1 ; done'
 exec sleep 5
 
 mkdir testrepo

cmd/pint/tests/0041_watch.txt

@@ -0,0 +1,26 @@
+exec bash -x ./test.sh &
+
+pint.ok --no-color -l debug watch --interval=10s --listen=:6041 --pidfile=pint.pid rules
+! stdout .
+cmp stderr stderr.txt
+grep '^pint_check_iterations_total 2$' curl.txt
+
+-- test.sh --
+sleep 15
+curl -so curl.txt http://127.0.0.1:6041/metrics
+cat pint.pid | xargs kill
+
+-- stderr.txt --
+level=info msg="Pidfile created" path=pint.pid
+level=info msg="Started HTTP server" address=:6041
+level=info msg="Will continuously run checks until terminated" interval=10s
+level=debug msg="Clearing cache"
+level=debug msg="Running checks"
+level=error msg="Got an error when running checks" error="no matching files"
+level=debug msg="Clearing cache"
+level=debug msg="Running checks"
+level=error msg="Got an error when running checks" error="no matching files"
+level=info msg="Shutting down"
+level=info msg="Waiting for all background tasks to finish"
+level=info msg="Background worker finished"
+level=info msg="Pidfile removed" path=pint.pid

cmd/pint/tests/0042_watch_metrics.txt

@@ -0,0 +1,152 @@
+exec bash -x ./test.sh &
+
+pint.ok watch --listen=:6042 --pidfile=pint.pid rules
+cmp curl.txt metrics.txt
+
+-- test.sh --
+sleep 5
+curl -s http://127.0.0.1:6042/metrics | grep -vE 'process_|go_info' | perl -pe "s/^([a-zA-Z].+)[ ]([0-9\.\-\+eE]+)$/\1/g" > curl.txt
+cat pint.pid | xargs kill
+
+-- rules/1.yml --
+- record: broken
+  expr: foo / count())
+
+- record: aggregate
+  expr: sum(foo) without(job)
+
+- alert: comparison
+  expr: foo
+
+-- .pint.hcl --
+rule {
+    match {
+      kind = "recording"
+    }
+    aggregate ".+" {
+        keep = [ "job" ]
+    }
+}
+
+-- metrics.txt --
+# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
+# TYPE go_gc_duration_seconds summary
+go_gc_duration_seconds{quantile="0"}
+go_gc_duration_seconds{quantile="0.25"}
+go_gc_duration_seconds{quantile="0.5"}
+go_gc_duration_seconds{quantile="0.75"}
+go_gc_duration_seconds{quantile="1"}
+go_gc_duration_seconds_sum
+go_gc_duration_seconds_count
+# HELP go_goroutines Number of goroutines that currently exist.
+# TYPE go_goroutines gauge
+go_goroutines
+# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+# TYPE go_memstats_alloc_bytes gauge
+go_memstats_alloc_bytes
+# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
+# TYPE go_memstats_alloc_bytes_total counter
+go_memstats_alloc_bytes_total
+# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
+# TYPE go_memstats_buck_hash_sys_bytes gauge
+go_memstats_buck_hash_sys_bytes
+# HELP go_memstats_frees_total Total number of frees.
+# TYPE go_memstats_frees_total counter
+go_memstats_frees_total
+# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started.
+# TYPE go_memstats_gc_cpu_fraction gauge
+go_memstats_gc_cpu_fraction
+# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
+# TYPE go_memstats_gc_sys_bytes gauge
+go_memstats_gc_sys_bytes
+# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
+# TYPE go_memstats_heap_alloc_bytes gauge
+go_memstats_heap_alloc_bytes
+# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
+# TYPE go_memstats_heap_idle_bytes gauge
+go_memstats_heap_idle_bytes
+# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
+# TYPE go_memstats_heap_inuse_bytes gauge
+go_memstats_heap_inuse_bytes
+# HELP go_memstats_heap_objects Number of allocated objects.
+# TYPE go_memstats_heap_objects gauge
+go_memstats_heap_objects
+# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS.
+# TYPE go_memstats_heap_released_bytes gauge
+go_memstats_heap_released_bytes
+# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
+# TYPE go_memstats_heap_sys_bytes gauge
+go_memstats_heap_sys_bytes
+# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
+# TYPE go_memstats_last_gc_time_seconds gauge
+go_memstats_last_gc_time_seconds
+# HELP go_memstats_lookups_total Total number of pointer lookups.
+# TYPE go_memstats_lookups_total counter
+go_memstats_lookups_total
+# HELP go_memstats_mallocs_total Total number of mallocs.
+# TYPE go_memstats_mallocs_total counter
+go_memstats_mallocs_total
+# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
+# TYPE go_memstats_mcache_inuse_bytes gauge
+go_memstats_mcache_inuse_bytes
+# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
+# TYPE go_memstats_mcache_sys_bytes gauge
+go_memstats_mcache_sys_bytes
+# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
+# TYPE go_memstats_mspan_inuse_bytes gauge
+go_memstats_mspan_inuse_bytes
+# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
+# TYPE go_memstats_mspan_sys_bytes gauge
+go_memstats_mspan_sys_bytes
+# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
+# TYPE go_memstats_next_gc_bytes gauge
+go_memstats_next_gc_bytes
+# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
+# TYPE go_memstats_other_sys_bytes gauge
+go_memstats_other_sys_bytes
+# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
+# TYPE go_memstats_stack_inuse_bytes gauge
+go_memstats_stack_inuse_bytes
+# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
+# TYPE go_memstats_stack_sys_bytes gauge
+go_memstats_stack_sys_bytes
+# HELP go_memstats_sys_bytes Number of bytes obtained from system.
+# TYPE go_memstats_sys_bytes gauge
+go_memstats_sys_bytes
+# HELP go_threads Number of OS threads created.
+# TYPE go_threads gauge
+go_threads
+# HELP pint_check_duration_seconds How long did a check took to complete
+# TYPE pint_check_duration_seconds summary
+pint_check_duration_seconds_sum{check="alerts/for"}
+pint_check_duration_seconds_count{check="alerts/for"}
+pint_check_duration_seconds_sum{check="alerts/template"}
+pint_check_duration_seconds_count{check="alerts/template"}
+pint_check_duration_seconds_sum{check="promql/aggregate"}
+pint_check_duration_seconds_count{check="promql/aggregate"}
+pint_check_duration_seconds_sum{check="promql/comparison"}
+pint_check_duration_seconds_count{check="promql/comparison"}
+pint_check_duration_seconds_sum{check="promql/syntax"}
+pint_check_duration_seconds_count{check="promql/syntax"}
+# HELP pint_check_iterations_total Total number of completed check iterations since pint start
+# TYPE pint_check_iterations_total counter
+pint_check_iterations_total
+# HELP pint_problem Prometheus rule problem reported by pint
+# TYPE pint_problem gauge
+pint_problem{kind="alerting",lines="8",name="comparison",problem="alert query doesn't have any condition, it will always fire if the metric exists",reporter="promql/comparison",severity="warning"}
+pint_problem{kind="recording",lines="2",name="broken",problem="syntax error: no arguments for aggregate expression provided",reporter="promql/syntax",severity="fatal"}
+pint_problem{kind="recording",lines="5",name="aggregate",problem="job label is required and should be preserved when aggregating \"^.+$\" rules, remove job from without()",reporter="promql/aggregate",severity="warning"}
+# HELP prometheus_template_text_expansion_failures_total The total number of template text expansion failures.
+# TYPE prometheus_template_text_expansion_failures_total counter
+prometheus_template_text_expansion_failures_total
+# HELP prometheus_template_text_expansions_total The total number of template text expansions.
+# TYPE prometheus_template_text_expansions_total counter
+prometheus_template_text_expansions_total
+# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
+# TYPE promhttp_metric_handler_requests_in_flight gauge
+promhttp_metric_handler_requests_in_flight
+# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
+# TYPE promhttp_metric_handler_requests_total counter
+promhttp_metric_handler_requests_total{code="200"}
+promhttp_metric_handler_requests_total{code="500"}
+promhttp_metric_handler_requests_total{code="503"}

cmd/pint/tests/0043_watch_cancel.txt

@@ -0,0 +1,80 @@
+exec bash -x ./prometheus.sh &
+exec bash -c 'while [ ! -f prometheus.pid ]; do sleep 1 ; done'
+
+exec bash -x ./test.sh &
+
+pint.ok --no-color watch --interval=1h --listen=:6043 --pidfile=pint.pid rules
+! stdout .
+stderr 'level=info msg="Shutting down"'
+stderr 'level=error msg="Failed to query Prometheus configuration" error="Get \\"http://127.0.0.1:7043/api/v1/status/config\\": context canceled" uri=http://127.0.0.1:7043'
+stderr 'level=error msg="Query failed" error="Post \\"http://127.0.0.1:7043/api/v1/query\\": context canceled" query=count\(foo\) uri=http://127.0.0.1:7043'
+stderr 'level=info msg="Waiting for all background tasks to finish"'
+stderr 'level=info msg="Background worker finished"'
+
+-- test.sh --
+sleep 5
+cat pint.pid | xargs kill
+cat prometheus.pid | xargs kill
+
+-- rules/1.yml --
+- record: aggregate
+  expr: sum(foo) without(job)
+
+-- .pint.hcl --
+prometheus "slow" {
+  uri     = "http://127.0.0.1:7043"
+  timeout = "2m"
+}
+
+-- prometheus.go --
+package main
+
+import (
+	"context"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+)
+
+func main() {
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+        w.WriteHeader(500)
+        time.Sleep(time.Second*30)
+		io.WriteString(w, "Fatal error")
+	})
+
+	listener, err := net.Listen("tcp", "127.0.0.1:7043")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	server := &http.Server{
+		Addr: "127.0.0.1:7043",
+	}
+
+	go func() {
+		_ = server.Serve(listener)
+	}()
+
+	pid := os.Getpid()
+	err = os.WriteFile("prometheus.pid", []byte(strconv.Itoa(pid)), 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+	<-stop
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	server.Shutdown(ctx)
+}
+
+-- prometheus.sh --
+env GOCACHE=$TMPDIR go run prometheus.go

cmd/pint/watch.go

@@ -9,6 +9,7 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -244,7 +245,7 @@ func (c *problemCollector) Collect(ch chan<- prometheus.Metric) {
 			1,
 			kind,
 			name,
-			report.Problem.Severity.String(),
+			strings.ToLower(report.Problem.Severity.String()),
 			report.Problem.Reporter,
 			report.Problem.Text,
 			output.FormatLineRangeString(report.Problem.Lines),