diff --git a/cmd/pint/tests/0054_watch_metrics_prometheus.txt b/cmd/pint/tests/0054_watch_metrics_prometheus.txt index 5514b051..7b4cbba2 100644 --- a/cmd/pint/tests/0054_watch_metrics_prometheus.txt +++ b/cmd/pint/tests/0054_watch_metrics_prometheus.txt @@ -64,14 +64,14 @@ pint_check_iterations_total pint_last_run_time_seconds # HELP pint_problem Prometheus rule problem reported by pint # TYPE pint_problem gauge -pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="cound't run \"promql/rate\" checks due to \"prom1\" prometheus connection error: failed to query Prometheus config: server_error: server error: 500",reporter="promql/rate",severity="bug"} -pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="cound't run \"promql/rate\" checks due to \"prom2\" prometheus connection error: failed to query Prometheus config: Get \"http://127.0.0.1:1054/api/v1/status/config\": dial tcp 127.0.0.1:1054: connect: connection refused",reporter="promql/rate",severity="bug"} -pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="cound't run \"promql/series\" checks due to \"prom2\" prometheus connection error: Post \"http://127.0.0.1:1054/api/v1/query\": dial tcp 127.0.0.1:1054: connect: connection refused",reporter="promql/series",severity="bug"} -pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="query using prom1 failed with: bad_response: Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|y\"\n }Fatal error|..., bigger context ...|:\"bad_data\",\n \"error\":\"bogus query\"\n }Fatal error|...",reporter="promql/series",severity="bug"} -pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="cound't run \"promql/rate\" checks due to \"prom1\" prometheus connection error: failed to query Prometheus config: server_error: server error: 500",reporter="promql/rate",severity="bug"} -pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="cound't run \"promql/rate\" checks due to \"prom2\" prometheus connection error: failed to query Prometheus config: Get \"http://127.0.0.1:1054/api/v1/status/config\": dial tcp 127.0.0.1:1054: connect: connection refused",reporter="promql/rate",severity="bug"} -pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="cound't run \"promql/series\" checks due to \"prom2\" prometheus connection error: Post \"http://127.0.0.1:1054/api/v1/query\": dial tcp 127.0.0.1:1054: connect: connection refused",reporter="promql/series",severity="bug"} -pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="query using prom1 failed with: bad_response: Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|y\"\n }Fatal error|..., bigger context ...|:\"bad_data\",\n \"error\":\"bogus query\"\n }Fatal error|...",reporter="promql/series",severity="bug"} +pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="cound't run \"promql/rate\" checks due to \"prom1\" on http://127.0.0.1:7054 connection error: failed to query Prometheus config: server_error: server error: 500",reporter="promql/rate",severity="bug"} +pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="cound't run \"promql/rate\" checks due to \"prom2\" on http://127.0.0.1:1054 connection error: failed to query Prometheus config: Get \"http://127.0.0.1:1054/api/v1/status/config\": dial tcp 127.0.0.1:1054: connect: connection refused",reporter="promql/rate",severity="bug"} +pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="cound't run \"promql/series\" checks due to \"prom2\" on http://127.0.0.1:1054 connection error: Post \"http://127.0.0.1:1054/api/v1/query\": dial tcp 127.0.0.1:1054: connect: connection refused",reporter="promql/series",severity="bug"} +pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="query using \"prom1\" on http://127.0.0.1:7054 failed with: bad_response: Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|y\"\n }Fatal error|..., bigger context ...|:\"bad_data\",\n \"error\":\"bogus query\"\n }Fatal error|...",reporter="promql/series",severity="bug"} +pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="cound't run \"promql/rate\" checks due to \"prom1\" on http://127.0.0.1:7054 connection error: failed to query Prometheus config: server_error: server error: 500",reporter="promql/rate",severity="bug"} +pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="cound't run \"promql/rate\" checks due to \"prom2\" on http://127.0.0.1:1054 connection error: failed to query Prometheus config: Get \"http://127.0.0.1:1054/api/v1/status/config\": dial tcp 127.0.0.1:1054: connect: connection refused",reporter="promql/rate",severity="bug"} +pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="cound't run \"promql/series\" checks due to \"prom2\" on http://127.0.0.1:1054 connection error: Post \"http://127.0.0.1:1054/api/v1/query\": dial tcp 127.0.0.1:1054: connect: connection refused",reporter="promql/series",severity="bug"} +pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="query using \"prom1\" on http://127.0.0.1:7054 failed with: bad_response: Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|y\"\n }Fatal error|..., bigger context ...|:\"bad_data\",\n \"error\":\"bogus query\"\n }Fatal error|...",reporter="promql/series",severity="bug"} pint_problem{filename="rules/1.yml",kind="recording",name="broken",problem="syntax error: no arguments for aggregate expression provided",reporter="promql/syntax",severity="fatal"} # HELP pint_problems Total number of problems reported by pint # TYPE pint_problems gauge diff --git a/cmd/pint/tests/0055_prometheus_failover.txt b/cmd/pint/tests/0055_prometheus_failover.txt index 94c8b0c5..f8115755 100644 --- a/cmd/pint/tests/0055_prometheus_failover.txt +++ b/cmd/pint/tests/0055_prometheus_failover.txt @@ -5,9 +5,8 @@ pint.ok --no-color lint rules ! stdout . stderr 'level=error msg="Query failed" error="Post \\"http://127.0.0.1:1055/api/v1/query\\": dial tcp 127.0.0.1:1055: connect: connection refused" query=count\(foo\) uri=http://127.0.0.1:1055' stderr 'level=error msg="Failed to query Prometheus configuration" error="Get \\"http://127.0.0.1:1055/api/v1/status/config\\": dial tcp 127.0.0.1:1055: connect: connection refused" uri=http://127.0.0.1:1055' -stderr 'rules/1.yml:2: query using prom completed without any results for foo \(promql/series\)' - -exec sh -c 'cat prometheus.pid | xargs kill' +stderr 'rules/1.yml:2: query using "prom" on http://127.0.0.1:7055 completed without any results for foo \(promql/series\)' +exec bash -c 'cat prometheus.pid | xargs kill' -- rules/1.yml -- - record: aggregate diff --git a/cmd/pint/tests/0057_watch_metrics_prometheus_ignore.txt b/cmd/pint/tests/0057_watch_metrics_prometheus_ignore.txt index a4a06c02..5db0b2c3 100644 --- a/cmd/pint/tests/0057_watch_metrics_prometheus_ignore.txt +++ b/cmd/pint/tests/0057_watch_metrics_prometheus_ignore.txt @@ -64,8 +64,8 @@ pint_check_iterations_total pint_last_run_time_seconds # HELP pint_problem Prometheus rule problem reported by pint # TYPE pint_problem gauge -pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="query using prom1 failed with: bad_response: Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|y\"\n }Fatal error|..., bigger context ...|:\"bad_data\",\n \"error\":\"bogus query\"\n }Fatal error|...",reporter="promql/series",severity="bug"} -pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="query using prom1 failed with: bad_response: Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|y\"\n }Fatal error|..., bigger context ...|:\"bad_data\",\n \"error\":\"bogus query\"\n }Fatal error|...",reporter="promql/series",severity="bug"} +pint_problem{filename="rules/1.yml",kind="alerting",name="comparison",problem="query using \"prom1\" on http://127.0.0.1:7057 failed with: bad_response: Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|y\"\n }Fatal error|..., bigger context ...|:\"bad_data\",\n \"error\":\"bogus query\"\n }Fatal error|...",reporter="promql/series",severity="bug"} +pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",problem="query using \"prom1\" on http://127.0.0.1:7057 failed with: bad_response: Unmarshal: there are bytes left after unmarshal, error found in #10 byte of ...|y\"\n }Fatal error|..., bigger context ...|:\"bad_data\",\n \"error\":\"bogus query\"\n }Fatal error|...",reporter="promql/series",severity="bug"} pint_problem{filename="rules/1.yml",kind="recording",name="broken",problem="syntax error: no arguments for aggregate expression provided",reporter="promql/syntax",severity="fatal"} # HELP pint_problems Total number of problems reported by pint # TYPE pint_problems gauge diff --git a/docs/changelog.md b/docs/changelog.md index 2f7722aa..556dc41c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -19,6 +19,7 @@ ### Changed - Added `filename` label to `pint_problem` metric - #170. +- Include Prometheus server URI in reported problems. ### Fixed diff --git a/internal/checks/alerts_count.go b/internal/checks/alerts_count.go index 220e89fc..39847974 100644 --- a/internal/checks/alerts_count.go +++ b/internal/checks/alerts_count.go @@ -107,7 +107,7 @@ func (c AlertsCheck) Check(ctx context.Context, rule parser.Rule) (problems []Pr Fragment: rule.AlertingRule.Expr.Value.Value, Lines: lines, Reporter: c.Reporter(), - Text: fmt.Sprintf("query using %s would trigger %d alert(s) in the last %s", c.prom.Name(), alerts, output.HumanizeDuration(delta)), + Text: fmt.Sprintf("query using %q on %s would trigger %d alert(s) in the last %s", c.prom.Name(), qr.URI, alerts, output.HumanizeDuration(delta)), Severity: Information, }) return diff --git a/internal/checks/alerts_count_test.go b/internal/checks/alerts_count_test.go index a62fe616..2df30369 100644 --- a/internal/checks/alerts_count_test.go +++ b/internal/checks/alerts_count_test.go @@ -101,7 +101,7 @@ func TestAlertsCheck(t *testing.T) { Fragment: `up{job="foo"} == 0`, Lines: []int{2}, Reporter: "alerts/count", - Text: "query using prom failed with: bad_data: unhandled path", + Text: fmt.Sprintf(`query using "prom" on %s/400/ failed with: bad_data: unhandled path`, srv.URL), Severity: checks.Bug, }, }, @@ -115,7 +115,7 @@ func TestAlertsCheck(t *testing.T) { Fragment: `up{job="foo"} == 0`, Lines: []int{2}, Reporter: "alerts/count", - Text: `cound't run "alerts/count" checks due to "prom" prometheus connection error: Post "http://127.0.0.1/api/v1/query_range": dial tcp 127.0.0.1:80: connect: connection refused`, + Text: `cound't run "alerts/count" checks due to "prom" on http://127.0.0.1 connection error: Post "http://127.0.0.1/api/v1/query_range": dial tcp 127.0.0.1:80: connect: connection refused`, Severity: checks.Warning, }, }, @@ -129,7 +129,7 @@ func TestAlertsCheck(t *testing.T) { Fragment: `up{job="foo"} == 0`, Lines: []int{2}, Reporter: "alerts/count", - Text: "query using prom would trigger 0 alert(s) in the last 1d", + Text: fmt.Sprintf(`query using "prom" on %s/empty/ would trigger 0 alert(s) in the last 1d`, srv.URL), Severity: checks.Information, }, }, @@ -143,7 +143,7 @@ func TestAlertsCheck(t *testing.T) { Fragment: `up{job="foo"} == 0`, Lines: []int{2}, Reporter: "alerts/count", - Text: "query using prom would trigger 7 alert(s) in the last 1d", + Text: fmt.Sprintf(`query using "prom" on %s/alerts/ would trigger 7 alert(s) in the last 1d`, srv.URL), Severity: checks.Information, }, }, @@ -157,7 +157,7 @@ func TestAlertsCheck(t *testing.T) { Fragment: `up{job="foo"} == 0`, Lines: []int{2, 3}, Reporter: "alerts/count", - Text: "query using prom would trigger 1 alert(s) in the last 1d", + Text: fmt.Sprintf(`query using "prom" on %s/alerts/ would trigger 1 alert(s) in the last 1d`, srv.URL), Severity: checks.Information, }, }, @@ -174,7 +174,7 @@ func TestAlertsCheck(t *testing.T) { Fragment: `{__name__="up", job="foo"} == 0`, Lines: []int{3}, Reporter: "alerts/count", - Text: "query using prom would trigger 3 alert(s) in the last 1d", + Text: fmt.Sprintf(`query using "prom" on %s/alerts/ would trigger 3 alert(s) in the last 1d`, srv.URL), Severity: checks.Information, }, }, @@ -191,7 +191,7 @@ func TestAlertsCheck(t *testing.T) { Fragment: `{__name__=~"(up|foo)", job="foo"} == 0`, Lines: []int{3}, Reporter: "alerts/count", - Text: "query using prom would trigger 3 alert(s) in the last 1d", + Text: fmt.Sprintf(`query using "prom" on %s/alerts/ would trigger 3 alert(s) in the last 1d`, srv.URL), Severity: checks.Information, }, }, diff --git a/internal/checks/base.go b/internal/checks/base.go index 14836c2a..c502dcd9 100644 --- a/internal/checks/base.go +++ b/internal/checks/base.go @@ -108,9 +108,12 @@ type exprProblem struct { func textAndSeverityFromError(err error, reporter, prom string, s Severity) (text string, severity Severity) { if promapi.IsUnavailableError(err) { - text = fmt.Sprintf("cound't run %q checks due to %q prometheus connection error: %s", reporter, prom, err) - var perr *promapi.Error + text = fmt.Sprintf("cound't run %q checks due to %q connection error: %s", reporter, prom, err) + var perr *promapi.FailoverGroupError if errors.As(err, &perr) { + if uri := perr.URI(); uri != "" { + text = fmt.Sprintf("cound't run %q checks due to %q on %s connection error: %s", reporter, prom, uri, err) + } if perr.IsStrict() { severity = Bug } else { @@ -120,7 +123,11 @@ func textAndSeverityFromError(err error, reporter, prom string, s Severity) (tex severity = Warning } } else { - text = fmt.Sprintf("query using %s failed with: %s", prom, err) + text = fmt.Sprintf("query using %q failed with: %s", prom, err) + var perr *promapi.FailoverGroupError + if errors.As(err, &perr) { + text = fmt.Sprintf("query using %q on %s failed with: %s", prom, perr.URI(), err) + } severity = s } return diff --git a/internal/checks/promql_rate.go b/internal/checks/promql_rate.go index f8ca0f7a..f3644fce 100644 --- a/internal/checks/promql_rate.go +++ b/internal/checks/promql_rate.go @@ -39,7 +39,7 @@ func (c RateCheck) Check(ctx context.Context, rule parser.Rule) (problems []Prob return } - scrapeInterval, err := c.getScrapeInterval(ctx) + cfg, err := c.prom.Config(ctx) if err != nil { text, severity := textAndSeverityFromError(err, c.Reporter(), c.prom.Name(), Bug) problems = append(problems, Problem{ @@ -52,7 +52,7 @@ func (c RateCheck) Check(ctx context.Context, rule parser.Rule) (problems []Prob return } - for _, problem := range c.checkNode(expr.Query, scrapeInterval) { + for _, problem := range c.checkNode(expr.Query, cfg) { problems = append(problems, Problem{ Fragment: problem.expr, Lines: expr.Lines(), @@ -65,16 +65,7 @@ func (c RateCheck) Check(ctx context.Context, rule parser.Rule) (problems []Prob return } -func (c RateCheck) getScrapeInterval(ctx context.Context) (interval time.Duration, err error) { - var cfg *promapi.PrometheusConfig - cfg, err = c.prom.Config(ctx) - if err != nil { - return - } - return cfg.Global.ScrapeInterval, nil -} - -func (c RateCheck) checkNode(node *parser.PromQLNode, scrapeInterval time.Duration) (problems []exprProblem) { +func (c RateCheck) checkNode(node *parser.PromQLNode, cfg *promapi.ConfigResult) (problems []exprProblem) { if n, ok := node.Node.(*promParser.Call); ok && (n.Func.Name == "rate" || n.Func.Name == "irate") { var minIntervals int var recIntervals int @@ -88,17 +79,19 @@ func (c RateCheck) checkNode(node *parser.PromQLNode, scrapeInterval time.Durati } for _, arg := range n.Args { if m, ok := arg.(*promParser.MatrixSelector); ok { - if m.Range < scrapeInterval*time.Duration(minIntervals) { + if m.Range < cfg.Config.Global.ScrapeInterval*time.Duration(minIntervals) { p := exprProblem{ - expr: node.Expr, - text: fmt.Sprintf("duration for %s() must be at least %d x scrape_interval, %s is using %s scrape_interval", n.Func.Name, minIntervals, c.prom.Name(), output.HumanizeDuration(scrapeInterval)), + expr: node.Expr, + text: fmt.Sprintf("duration for %s() must be at least %d x scrape_interval, %q on %s is using %s scrape_interval", + n.Func.Name, minIntervals, c.prom.Name(), cfg.URI, output.HumanizeDuration(cfg.Config.Global.ScrapeInterval)), severity: Bug, } problems = append(problems, p) - } else if m.Range < scrapeInterval*time.Duration(recIntervals) { + } else if m.Range < cfg.Config.Global.ScrapeInterval*time.Duration(recIntervals) { p := exprProblem{ - expr: node.Expr, - text: fmt.Sprintf("duration for %s() is recommended to be at least %d x scrape_interval, %s is using %s scrape_interval", n.Func.Name, recIntervals, c.prom.Name(), output.HumanizeDuration(scrapeInterval)), + expr: node.Expr, + text: fmt.Sprintf("duration for %s() is recommended to be at least %d x scrape_interval, %q on %s is using %s scrape_interval", + n.Func.Name, recIntervals, c.prom.Name(), cfg.URI, output.HumanizeDuration(cfg.Config.Global.ScrapeInterval)), severity: Warning, } problems = append(problems, p) @@ -108,7 +101,7 @@ func (c RateCheck) checkNode(node *parser.PromQLNode, scrapeInterval time.Durati } for _, child := range node.Children { - problems = append(problems, c.checkNode(child, scrapeInterval)...) + problems = append(problems, c.checkNode(child, cfg)...) } return diff --git a/internal/checks/promql_rate_test.go b/internal/checks/promql_rate_test.go index a96e03b0..cae3ad94 100644 --- a/internal/checks/promql_rate_test.go +++ b/internal/checks/promql_rate_test.go @@ -59,7 +59,7 @@ func TestRateCheck(t *testing.T) { Fragment: "rate(foo[1m])", Lines: []int{2}, Reporter: "promql/rate", - Text: "duration for rate() must be at least 2 x scrape_interval, prom is using 1m scrape_interval", + Text: fmt.Sprintf(`duration for rate() must be at least 2 x scrape_interval, "prom" on %s is using 1m scrape_interval`, srv.URL+"/1m/"), Severity: checks.Bug, }, }, @@ -73,7 +73,7 @@ func TestRateCheck(t *testing.T) { Fragment: "rate(foo[3m])", Lines: []int{2}, Reporter: "promql/rate", - Text: "duration for rate() is recommended to be at least 4 x scrape_interval, prom is using 1m scrape_interval", + Text: fmt.Sprintf(`duration for rate() is recommended to be at least 4 x scrape_interval, "prom" on %s is using 1m scrape_interval`, srv.URL+"/1m/"), Severity: checks.Warning, }, }, @@ -92,7 +92,7 @@ func TestRateCheck(t *testing.T) { Fragment: "irate(foo[1m])", Lines: []int{2}, Reporter: "promql/rate", - Text: "duration for irate() must be at least 2 x scrape_interval, prom is using 1m scrape_interval", + Text: fmt.Sprintf(`duration for irate() must be at least 2 x scrape_interval, "prom" on %s is using 1m scrape_interval`, srv.URL+"/1m/"), Severity: checks.Bug, }, }, @@ -106,7 +106,7 @@ func TestRateCheck(t *testing.T) { Fragment: "irate(foo[2m])", Lines: []int{2}, Reporter: "promql/rate", - Text: "duration for irate() is recommended to be at least 3 x scrape_interval, prom is using 1m scrape_interval", + Text: fmt.Sprintf(`duration for irate() is recommended to be at least 3 x scrape_interval, "prom" on %s is using 1m scrape_interval`, srv.URL+"/1m/"), Severity: checks.Warning, }, }, @@ -139,7 +139,7 @@ func TestRateCheck(t *testing.T) { Fragment: `irate({__name__="foo"}[2m])`, Lines: []int{3}, Reporter: "promql/rate", - Text: "duration for irate() is recommended to be at least 3 x scrape_interval, prom is using 1m scrape_interval", + Text: fmt.Sprintf(`duration for irate() is recommended to be at least 3 x scrape_interval, "prom" on %s is using 1m scrape_interval`, srv.URL+"/1m/"), Severity: checks.Warning, }, }, @@ -156,7 +156,7 @@ func TestRateCheck(t *testing.T) { Fragment: `irate({__name__=~"(foo|bar)_total"}[2m])`, Lines: []int{3}, Reporter: "promql/rate", - Text: "duration for irate() is recommended to be at least 3 x scrape_interval, prom is using 1m scrape_interval", + Text: fmt.Sprintf(`duration for irate() is recommended to be at least 3 x scrape_interval, "prom" on %s is using 1m scrape_interval`, srv.URL+"/1m/"), Severity: checks.Warning, }, }, @@ -180,14 +180,14 @@ func TestRateCheck(t *testing.T) { Fragment: "rate(foo[3m])", Lines: []int{2}, Reporter: "promql/rate", - Text: "duration for rate() is recommended to be at least 4 x scrape_interval, prom is using 1m scrape_interval", + Text: fmt.Sprintf(`duration for rate() is recommended to be at least 4 x scrape_interval, "prom" on %s is using 1m scrape_interval`, srv.URL+"/1m/"), Severity: checks.Warning, }, { Fragment: "rate(bar[1m])", Lines: []int{2}, Reporter: "promql/rate", - Text: "duration for rate() must be at least 2 x scrape_interval, prom is using 1m scrape_interval", + Text: fmt.Sprintf(`duration for rate() must be at least 2 x scrape_interval, "prom" on %s is using 1m scrape_interval`, srv.URL+"/1m/"), Severity: checks.Bug, }, }, @@ -201,7 +201,7 @@ func TestRateCheck(t *testing.T) { Fragment: "rate(foo[5m])", Lines: []int{2}, Reporter: "promql/rate", - Text: `cound't run "promql/rate" checks due to "prom" prometheus connection error: failed to query Prometheus config: server_error: server error: 500`, + Text: fmt.Sprintf(`cound't run "promql/rate" checks due to "prom" on %s/error/ connection error: failed to query Prometheus config: server_error: server error: 500`, srv.URL), Severity: checks.Bug, }, }, @@ -215,7 +215,7 @@ func TestRateCheck(t *testing.T) { Fragment: "rate(foo[5m])", Lines: []int{2}, Reporter: "promql/rate", - Text: `query using prom failed with: failed to query Prometheus config: bad_data: unhandled path`, + Text: fmt.Sprintf(`query using "prom" on %s failed with: failed to query Prometheus config: bad_data: unhandled path`, srv.URL), Severity: checks.Bug, }, }, @@ -229,7 +229,7 @@ func TestRateCheck(t *testing.T) { Fragment: "rate(foo[5m])", Lines: []int{2}, Reporter: "promql/rate", - Text: fmt.Sprintf("cound't run \"promql/rate\" checks due to \"prom\" prometheus connection error: failed to decode config data in %s/badYaml/ response: yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into promapi.PrometheusConfig", srv.URL), + Text: fmt.Sprintf("cound't run \"promql/rate\" checks due to \"prom\" on %s/badYaml/ connection error: failed to decode config data in %s/badYaml/ response: yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into promapi.PrometheusConfig", srv.URL, srv.URL), Severity: checks.Bug, }, }, @@ -243,7 +243,7 @@ func TestRateCheck(t *testing.T) { Fragment: "rate(foo[5m])", Lines: []int{2}, Reporter: "promql/rate", - Text: `cound't run "promql/rate" checks due to "prom" prometheus connection error: failed to query Prometheus config: Get "http:///api/v1/status/config": http: no Host in request URL`, + Text: `cound't run "promql/rate" checks due to "prom" on http:// connection error: failed to query Prometheus config: Get "http:///api/v1/status/config": http: no Host in request URL`, Severity: checks.Warning, }, }, @@ -262,7 +262,7 @@ func TestRateCheck(t *testing.T) { Fragment: "irate(foo[2m])", Lines: []int{2}, Reporter: "promql/rate", - Text: "duration for irate() is recommended to be at least 3 x scrape_interval, prom is using 1m scrape_interval", + Text: fmt.Sprintf(`duration for irate() is recommended to be at least 3 x scrape_interval, "prom" on %s is using 1m scrape_interval`, srv.URL+"/default/"), Severity: checks.Warning, }, }, diff --git a/internal/checks/promql_series.go b/internal/checks/promql_series.go index 682b3d05..2a83070a 100644 --- a/internal/checks/promql_series.go +++ b/internal/checks/promql_series.go @@ -95,7 +95,7 @@ func (c SeriesCheck) countSeries(ctx context.Context, expr parser.PromQLExpr, se Fragment: selector.String(), Lines: expr.Lines(), Reporter: c.Reporter(), - Text: fmt.Sprintf("query using %s completed without any results for %s", c.prom.Name(), selector.String()), + Text: fmt.Sprintf("query using %q on %s completed without any results for %s", c.prom.Name(), qr.URI, selector.String()), Severity: Warning, }) return diff --git a/internal/checks/promql_series_test.go b/internal/checks/promql_series_test.go index da65f1fa..3c4c8e10 100644 --- a/internal/checks/promql_series_test.go +++ b/internal/checks/promql_series_test.go @@ -1,6 +1,7 @@ package checks_test import ( + "fmt" "net/http" "net/http/httptest" "testing" @@ -119,7 +120,7 @@ func TestSeriesCheck(t *testing.T) { Fragment: "foo", Lines: []int{2}, Reporter: "promql/series", - Text: "query using prom failed with: bad_data: unhandled query", + Text: fmt.Sprintf(`query using "prom" on %s failed with: bad_data: unhandled query`, srv.URL), Severity: checks.Bug, }, }, @@ -133,7 +134,7 @@ func TestSeriesCheck(t *testing.T) { Fragment: "foo", Lines: []int{2}, Reporter: "promql/series", - Text: `cound't run "promql/series" checks due to "prom" prometheus connection error: Post "http:///api/v1/query": http: no Host in request URL`, + Text: `cound't run "promql/series" checks due to "prom" on http:// connection error: Post "http:///api/v1/query": http: no Host in request URL`, Severity: checks.Warning, }, }, @@ -147,7 +148,7 @@ func TestSeriesCheck(t *testing.T) { Fragment: "notfound", Lines: []int{2}, Reporter: "promql/series", - Text: "query using prom completed without any results for notfound", + Text: fmt.Sprintf(`query using "prom" on %s completed without any results for notfound`, srv.URL), Severity: checks.Warning, }, }, @@ -161,7 +162,7 @@ func TestSeriesCheck(t *testing.T) { Fragment: "notfound", Lines: []int{2}, Reporter: "promql/series", - Text: "query using prom completed without any results for notfound", + Text: fmt.Sprintf(`query using "prom" on %s completed without any results for notfound`, srv.URL), Severity: checks.Warning, }, }, @@ -175,7 +176,7 @@ func TestSeriesCheck(t *testing.T) { Fragment: "notfound", Lines: []int{2}, Reporter: "promql/series", - Text: "query using prom completed without any results for notfound", + Text: fmt.Sprintf(`query using "prom" on %s completed without any results for notfound`, srv.URL), Severity: checks.Warning, }, }, @@ -224,7 +225,7 @@ func TestSeriesCheck(t *testing.T) { Fragment: `found{job="notfound"}`, Lines: []int{2}, Reporter: "promql/series", - Text: `query using prom completed without any results for found{job="notfound"}`, + Text: fmt.Sprintf(`query using "prom" on %s completed without any results for found{job="notfound"}`, srv.URL), Severity: checks.Warning, }, }, @@ -238,7 +239,7 @@ func TestSeriesCheck(t *testing.T) { Fragment: "notfound", Lines: []int{2}, Reporter: "promql/series", - Text: "query using prom completed without any results for notfound", + Text: fmt.Sprintf(`query using "prom" on %s completed without any results for notfound`, srv.URL), Severity: checks.Warning, }, }, @@ -255,7 +256,7 @@ func TestSeriesCheck(t *testing.T) { Fragment: `{__name__="notfound",job="bar"}`, Lines: []int{3}, Reporter: "promql/series", - Text: `query using prom completed without any results for {__name__="notfound",job="bar"}`, + Text: fmt.Sprintf(`query using "prom" on %s completed without any results for {__name__="notfound",job="bar"}`, srv.URL), Severity: checks.Warning, }, }, @@ -300,7 +301,7 @@ func TestSeriesCheck(t *testing.T) { Fragment: `notfound`, Lines: []int{4}, Reporter: "promql/series", - Text: `query using prom completed without any results for notfound`, + Text: fmt.Sprintf(`query using "prom" on %s completed without any results for notfound`, srv.URL), Severity: checks.Warning, }, }, diff --git a/internal/checks/promql_vector_matching_test.go b/internal/checks/promql_vector_matching_test.go index dd97552d..e9642d77 100644 --- a/internal/checks/promql_vector_matching_test.go +++ b/internal/checks/promql_vector_matching_test.go @@ -382,7 +382,7 @@ func TestVectorMatchingCheck(t *testing.T) { Fragment: "xxx/yyy", Lines: []int{2}, Reporter: "promql/vector_matching", - Text: `cound't run "promql/vector_matching" checks due to "prom" prometheus connection error: Post "http://127.0.0.1/api/v1/query": dial tcp 127.0.0.1:80: connect: connection refused`, + Text: `cound't run "promql/vector_matching" checks due to "prom" on http://127.0.0.1 connection error: Post "http://127.0.0.1/api/v1/query": dial tcp 127.0.0.1:80: connect: connection refused`, Severity: checks.Bug, }, }, @@ -396,7 +396,7 @@ func TestVectorMatchingCheck(t *testing.T) { Fragment: "xxx/yyy", Lines: []int{2}, Reporter: "promql/vector_matching", - Text: `cound't run "promql/vector_matching" checks due to "prom" prometheus connection error: Post "http://127.0.0.1/api/v1/query": dial tcp 127.0.0.1:80: connect: connection refused`, + Text: `cound't run "promql/vector_matching" checks due to "prom" on http://127.0.0.1 connection error: Post "http://127.0.0.1/api/v1/query": dial tcp 127.0.0.1:80: connect: connection refused`, Severity: checks.Warning, }, }, diff --git a/internal/checks/query_cost.go b/internal/checks/query_cost.go index 3644c4a6..43adedc8 100644 --- a/internal/checks/query_cost.go +++ b/internal/checks/query_cost.go @@ -82,7 +82,7 @@ func (c CostCheck) Check(ctx context.Context, rule parser.Rule) (problems []Prob Fragment: expr.Value.Value, Lines: expr.Lines(), Reporter: c.Reporter(), - Text: fmt.Sprintf("query using %s completed in %.2fs returning %d result(s)%s%s", c.prom.Name(), qr.DurationSeconds, series, estimate, above), + Text: fmt.Sprintf("query using %q on %s completed in %.2fs returning %d result(s)%s%s", c.prom.Name(), qr.URI, qr.DurationSeconds, series, estimate, above), Severity: severity, }) return diff --git a/internal/checks/query_cost_test.go b/internal/checks/query_cost_test.go index 1a5b0045..c2ff00c8 100644 --- a/internal/checks/query_cost_test.go +++ b/internal/checks/query_cost_test.go @@ -1,6 +1,7 @@ package checks_test import ( + "fmt" "net/http" "net/http/httptest" "regexp" @@ -90,7 +91,7 @@ func TestCostCheck(t *testing.T) { Fragment: "sum(foo)", Lines: []int{2}, Reporter: "query/cost", - Text: `RE:query using prom completed in 2\...s returning 0 result\(s\)`, + Text: fmt.Sprintf(`RE:query using "prom" on %s completed in 2\...s returning 0 result\(s\)`, srv.URL+"/empty/"), Severity: checks.Information, }, }, @@ -104,7 +105,7 @@ func TestCostCheck(t *testing.T) { Fragment: "sum(foo)", Lines: []int{2}, Reporter: "query/cost", - Text: `RE:cound't run "query/cost" checks due to "prom" prometheus connection error: Post "http://127.0.0.1:.+/empty/api/v1/query": context deadline exceeded`, + Text: fmt.Sprintf(`RE:cound't run "query/cost" checks due to "prom" on %s/empty/ connection error: Post "http://127.0.0.1:.+/empty/api/v1/query": context deadline exceeded`, srv.URL), Severity: checks.Bug, }, }, @@ -118,7 +119,7 @@ func TestCostCheck(t *testing.T) { Fragment: "sum(foo)", Lines: []int{2}, Reporter: "query/cost", - Text: "query using prom failed with: bad_data: unhandled path", + Text: fmt.Sprintf(`query using "prom" on %s/400/ failed with: bad_data: unhandled path`, srv.URL), Severity: checks.Bug, }, }, @@ -132,7 +133,7 @@ func TestCostCheck(t *testing.T) { Fragment: "sum(foo)", Lines: []int{2}, Reporter: "query/cost", - Text: `cound't run "query/cost" checks due to "prom" prometheus connection error: Post "http://127.0.0.1/api/v1/query": dial tcp 127.0.0.1:80: connect: connection refused`, + Text: `cound't run "query/cost" checks due to "prom" on http://127.0.0.1 connection error: Post "http://127.0.0.1/api/v1/query": dial tcp 127.0.0.1:80: connect: connection refused`, Severity: checks.Warning, }, }, @@ -146,7 +147,7 @@ func TestCostCheck(t *testing.T) { Fragment: "sum(foo)", Lines: []int{2}, Reporter: "query/cost", - Text: `RE:query using prom completed in 0\...s returning 1 result\(s\) with 4\.0KiB estimated memory usage`, + Text: fmt.Sprintf(`RE:query using "prom" on %s completed in 0\...s returning 1 result\(s\) with 4\.0KiB estimated memory usage`, srv.URL+"/1/"), Severity: checks.Information, }, }, @@ -160,7 +161,7 @@ func TestCostCheck(t *testing.T) { Fragment: "sum(foo)", Lines: []int{2}, Reporter: "query/cost", - Text: `RE:query using prom completed in 0\...s returning 7 result\(s\) with 707B estimated memory usage`, + Text: fmt.Sprintf(`RE:query using "prom" on %s completed in 0\...s returning 7 result\(s\) with 707B estimated memory usage`, srv.URL+"/7/"), Severity: checks.Information, }, }, @@ -174,7 +175,7 @@ func TestCostCheck(t *testing.T) { Fragment: "sum(foo)", Lines: []int{2}, Reporter: "query/cost", - Text: `RE:query using prom completed in 0\...s returning 7 result\(s\) with 7\.0MiB estimated memory usage`, + Text: fmt.Sprintf(`RE:query using "prom" on %s completed in 0\...s returning 7 result\(s\) with 7\.0MiB estimated memory usage`, srv.URL+"/7/"), Severity: checks.Information, }, }, @@ -188,7 +189,7 @@ func TestCostCheck(t *testing.T) { Fragment: "sum(foo)", Lines: []int{2}, Reporter: "query/cost", - Text: `RE:query using prom completed in 0\...s returning 7 result\(s\) with 7.0KiB estimated memory usage, maximum allowed series is 1`, + Text: fmt.Sprintf(`RE:query using "prom" on %s completed in 0\...s returning 7 result\(s\) with 7.0KiB estimated memory usage, maximum allowed series is 1`, srv.URL+"/7/"), Severity: checks.Bug, }, }, @@ -202,7 +203,7 @@ func TestCostCheck(t *testing.T) { Fragment: "sum(foo)", Lines: []int{2}, Reporter: "query/cost", - Text: `RE:query using prom completed in 0\...s returning 7 result\(s\), maximum allowed series is 5`, + Text: fmt.Sprintf(`RE:query using "prom" on %s completed in 0\...s returning 7 result\(s\), maximum allowed series is 5`, srv.URL+"/7/"), Severity: checks.Bug, }, }, @@ -216,7 +217,7 @@ func TestCostCheck(t *testing.T) { Fragment: "sum(foo)", Lines: []int{2}, Reporter: "query/cost", - Text: `RE:query using prom completed in 0\...s returning 7 result\(s\), maximum allowed series is 5`, + Text: fmt.Sprintf(`RE:query using "prom" on %s completed in 0\...s returning 7 result\(s\), maximum allowed series is 5`, srv.URL+"/7/"), Severity: checks.Information, }, }, @@ -233,7 +234,7 @@ func TestCostCheck(t *testing.T) { Fragment: `sum({__name__="foo"})`, Lines: []int{3}, Reporter: "query/cost", - Text: `RE:query using prom completed in 0\...s returning 7 result\(s\) with 707B estimated memory usage`, + Text: fmt.Sprintf(`RE:query using "prom" on %s completed in 0\...s returning 7 result\(s\) with 707B estimated memory usage`, srv.URL+"/7/"), Severity: checks.Information, }, }, diff --git a/internal/promapi/config.go b/internal/promapi/config.go index 79424bb4..8dbce828 100644 --- a/internal/promapi/config.go +++ b/internal/promapi/config.go @@ -20,7 +20,12 @@ type PrometheusConfig struct { Global ConfigSectionGlobal `yaml:"global"` } -func (p *Prometheus) Config(ctx context.Context) (*PrometheusConfig, error) { +type ConfigResult struct { + URI string + Config PrometheusConfig +} + +func (p *Prometheus) Config(ctx context.Context) (*ConfigResult, error) { log.Debug().Str("uri", p.uri).Msg("Query Prometheus configuration") key := "/api/v1/status/config" @@ -29,7 +34,7 @@ func (p *Prometheus) Config(ctx context.Context) (*PrometheusConfig, error) { if v, ok := p.cache.Get(key); ok { log.Debug().Str("key", key).Str("uri", p.uri).Msg("Config cache hit") - cfg := v.(PrometheusConfig) + cfg := v.(ConfigResult) return &cfg, nil } @@ -60,8 +65,10 @@ func (p *Prometheus) Config(ctx context.Context) (*PrometheusConfig, error) { cfg.Global.EvaluationInterval = time.Minute } + r := ConfigResult{URI: p.uri, Config: cfg} + log.Debug().Str("key", key).Str("uri", p.uri).Msg("Config cache miss") - p.cache.Add(key, cfg) + p.cache.Add(key, r) - return &cfg, nil + return &r, nil } diff --git a/internal/promapi/config_test.go b/internal/promapi/config_test.go index b934e24e..dd5a9243 100644 --- a/internal/promapi/config_test.go +++ b/internal/promapi/config_test.go @@ -64,7 +64,7 @@ func TestConfig(t *testing.T) { type testCaseT struct { prefix string timeout time.Duration - cfg promapi.PrometheusConfig + cfg promapi.ConfigResult err string runs int } @@ -82,24 +82,33 @@ func TestConfig(t *testing.T) { { prefix: "/default", timeout: time.Second, - cfg: defaults, - runs: 5, + cfg: promapi.ConfigResult{ + URI: srv.URL + "/default", + Config: defaults, + }, + runs: 5, }, { prefix: "/1m", timeout: time.Second, - cfg: defaults, - runs: 5, + cfg: promapi.ConfigResult{ + URI: srv.URL + "/1m", + Config: defaults, + }, + runs: 5, }, { prefix: "/30s", timeout: time.Second, - cfg: promapi.PrometheusConfig{ - Global: promapi.ConfigSectionGlobal{ - ScrapeInterval: time.Second * 30, - ScrapeTimeout: time.Second * 10, - EvaluationInterval: time.Minute, - ExternalLabels: nil, + cfg: promapi.ConfigResult{ + URI: srv.URL + "/30s", + Config: promapi.PrometheusConfig{ + Global: promapi.ConfigSectionGlobal{ + ScrapeInterval: time.Second * 30, + ScrapeTimeout: time.Second * 10, + EvaluationInterval: time.Minute, + ExternalLabels: nil, + }, }, }, runs: 1, @@ -125,8 +134,11 @@ func TestConfig(t *testing.T) { { prefix: "/once", timeout: time.Second, - cfg: defaults, - runs: 10, + cfg: promapi.ConfigResult{ + URI: srv.URL + "/once", + Config: defaults, + }, + runs: 10, }, // make sure /once fails on second query { diff --git a/internal/promapi/errors.go b/internal/promapi/errors.go index 6b5b0b5f..9195462c 100644 --- a/internal/promapi/errors.go +++ b/internal/promapi/errors.go @@ -10,23 +10,6 @@ import ( v1 "github.com/prometheus/client_golang/api/prometheus/v1" ) -type Error struct { - err error - isStrict bool -} - -func (e *Error) Unwrap() error { - return e.err -} - -func (e *Error) Error() string { - return e.err.Error() -} - -func (e *Error) IsStrict() bool { - return e.isStrict -} - func IsUnavailableError(err error) bool { var apiErr *v1.Error if ok := errors.As(err, &apiErr); ok { diff --git a/internal/promapi/failover.go b/internal/promapi/failover.go index afaef7e0..2e5d2cf0 100644 --- a/internal/promapi/failover.go +++ b/internal/promapi/failover.go @@ -5,6 +5,28 @@ import ( "time" ) +type FailoverGroupError struct { + err error + uri string + isStrict bool +} + +func (e *FailoverGroupError) Unwrap() error { + return e.err +} + +func (e *FailoverGroupError) Error() string { + return e.err.Error() +} + +func (e *FailoverGroupError) URI() string { + return e.uri +} + +func (e *FailoverGroupError) IsStrict() bool { + return e.isStrict +} + type FailoverGroup struct { name string servers []*Prometheus @@ -29,32 +51,47 @@ func (fg *FailoverGroup) ClearCache() { } } -func (fg *FailoverGroup) Config(ctx context.Context) (cfg *PrometheusConfig, err error) { +func (fg *FailoverGroup) Config(ctx context.Context) (cfg *ConfigResult, err error) { + var uri string for _, prom := range fg.servers { + uri = prom.uri cfg, err = prom.Config(ctx) - if err == nil || !IsUnavailableError(err) { + if err == nil { return } + if !IsUnavailableError(err) { + return cfg, &FailoverGroupError{err: err, uri: uri, isStrict: fg.strictErrors} + } } - return nil, &Error{err: err, isStrict: fg.strictErrors} + return nil, &FailoverGroupError{err: err, uri: uri, isStrict: fg.strictErrors} } func (fg *FailoverGroup) Query(ctx context.Context, expr string) (qr *QueryResult, err error) { + var uri string for _, prom := range fg.servers { + uri = prom.uri qr, err = prom.Query(ctx, expr) - if err == nil || !IsUnavailableError(err) { + if err == nil { return } + if !IsUnavailableError(err) { + return qr, &FailoverGroupError{err: err, uri: uri, isStrict: fg.strictErrors} + } } - return nil, &Error{err: err, isStrict: fg.strictErrors} + return nil, &FailoverGroupError{err: err, uri: uri, isStrict: fg.strictErrors} } func (fg *FailoverGroup) RangeQuery(ctx context.Context, expr string, start, end time.Time, step time.Duration) (rqr *RangeQueryResult, err error) { + var uri string for _, prom := range fg.servers { + uri = prom.uri rqr, err = prom.RangeQuery(ctx, expr, start, end, step) - if err == nil || !IsUnavailableError(err) { + if err == nil { return } + if !IsUnavailableError(err) { + return rqr, &FailoverGroupError{err: err, uri: uri, isStrict: fg.strictErrors} + } } - return nil, &Error{err: err, isStrict: fg.strictErrors} + return nil, &FailoverGroupError{err: err, uri: uri, isStrict: fg.strictErrors} } diff --git a/internal/promapi/query.go b/internal/promapi/query.go index 1e66ead8..1c2b37aa 100644 --- a/internal/promapi/query.go +++ b/internal/promapi/query.go @@ -12,6 +12,7 @@ import ( ) type QueryResult struct { + URI string Series model.Vector DurationSeconds float64 } @@ -53,6 +54,7 @@ func (p *Prometheus) Query(ctx context.Context, expr string) (*QueryResult, erro } qr := QueryResult{ + URI: p.uri, DurationSeconds: duration.Seconds(), } diff --git a/internal/promapi/query_test.go b/internal/promapi/query_test.go index 75adc0a3..6bb4f1b2 100644 --- a/internal/promapi/query_test.go +++ b/internal/promapi/query_test.go @@ -111,7 +111,7 @@ func TestQuery(t *testing.T) { type testCaseT struct { query string timeout time.Duration - series model.Vector + result promapi.QueryResult err string runs int } @@ -120,17 +120,23 @@ func TestQuery(t *testing.T) { { query: "empty", timeout: time.Second, - series: model.Vector{}, - runs: 5, + result: promapi.QueryResult{ + URI: srv.URL, + Series: model.Vector{}, + }, + runs: 5, }, { query: "single_result", timeout: time.Second, - series: model.Vector{ - &model.Sample{ - Metric: model.Metric{}, - Value: model.SampleValue(1), - Timestamp: model.Time(1614859502068), + result: promapi.QueryResult{ + URI: srv.URL, + Series: model.Vector{ + &model.Sample{ + Metric: model.Metric{}, + Value: model.SampleValue(1), + Timestamp: model.Time(1614859502068), + }, }, }, runs: 5, @@ -138,21 +144,24 @@ func TestQuery(t *testing.T) { { query: "three_results", timeout: time.Second, - series: model.Vector{ - &model.Sample{ - Metric: model.Metric{"instance": "1"}, - Value: model.SampleValue(1), - Timestamp: model.Time(1614859502068), - }, - &model.Sample{ - Metric: model.Metric{"instance": "2"}, - Value: model.SampleValue(2), - Timestamp: model.Time(1614859502168), - }, - &model.Sample{ - Metric: model.Metric{"instance": "3"}, - Value: model.SampleValue(3), - Timestamp: model.Time(1614859503000), + result: promapi.QueryResult{ + URI: srv.URL, + Series: model.Vector{ + &model.Sample{ + Metric: model.Metric{"instance": "1"}, + Value: model.SampleValue(1), + Timestamp: model.Time(1614859502068), + }, + &model.Sample{ + Metric: model.Metric{"instance": "2"}, + Value: model.SampleValue(2), + Timestamp: model.Time(1614859502168), + }, + &model.Sample{ + Metric: model.Metric{"instance": "3"}, + Value: model.SampleValue(3), + Timestamp: model.Time(1614859503000), + }, }, }, runs: 5, @@ -178,11 +187,14 @@ func TestQuery(t *testing.T) { { query: "once", timeout: time.Second, - series: model.Vector{ - &model.Sample{ - Metric: model.Metric{}, - Value: model.SampleValue(1), - Timestamp: model.Time(1614859502068), + result: promapi.QueryResult{ + URI: srv.URL, + Series: model.Vector{ + &model.Sample{ + Metric: model.Metric{}, + Value: model.SampleValue(1), + Timestamp: model.Time(1614859502068), + }, }, }, runs: 5, @@ -213,7 +225,8 @@ func TestQuery(t *testing.T) { assert.NoError(err) } if qr != nil { - assert.Equal(qr.Series, tc.series) + assert.Equal(tc.result.URI, qr.URI) + assert.Equal(tc.result.Series, qr.Series) } wg.Done() }() diff --git a/internal/promapi/range.go b/internal/promapi/range.go index 2d730b44..30582633 100644 --- a/internal/promapi/range.go +++ b/internal/promapi/range.go @@ -15,6 +15,7 @@ import ( ) type RangeQueryResult struct { + URI string Samples []*model.SampleStream Start time.Time End time.Time @@ -74,6 +75,7 @@ func (p *Prometheus) RangeQuery(ctx context.Context, expr string, start, end tim } qr := RangeQueryResult{ + URI: p.uri, DurationSeconds: duration.Seconds(), Start: start, End: end,