From eeb16289fea8213098f59159b196d59f017b4821 Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Thu, 9 Jun 2022 17:41:58 +0200 Subject: [PATCH] Revert unwrapped rate aggregation to previous implementation This PR reverts the implementation done in #5013 to the original implementation that sums the extracted values from the log lines instead of treating them like a Prometheus counter metric. Signed-off-by: Christian Haudum --- CHANGELOG.md | 3 ++- docs/sources/logql/metric_queries.md | 2 +- pkg/logql/engine_test.go | 40 +++++++++++++++++++++++----- pkg/logql/range_vector.go | 9 +++++-- 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 429c7f8e681bf..d23a7442da365 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## Main -* [6099](https://github.com/grafana/loki/pull/6099/files) **cstyan**: Drop lines with malformed JSON in Promtail JSON pipeline stage +* [6361](https://github.com/grafana/loki/pull/6361) **chaudum**: Sum values in unwrapped rate aggregation instead of treating them as counter +* [6099](https://github.com/grafana/loki/pull/6099) **cstyan**: Drop lines with malformed JSON in Promtail JSON pipeline stage * [6136](https://github.com/grafana/loki/pull/6136) **periklis**: Add support for alertmanager header authorization * [6102](https://github.com/grafana/loki/pull/6102) **timchenko-a**: Add multi-tenancy support to lambda-promtail * [5971](https://github.com/grafana/loki/pull/5971) **kavirajk**: Record statistics about metadata queries such as labels and series queries in `metrics.go` as well diff --git a/docs/sources/logql/metric_queries.md b/docs/sources/logql/metric_queries.md index c34da49b67e78..e5ad7315db170 100644 --- a/docs/sources/logql/metric_queries.md +++ b/docs/sources/logql/metric_queries.md @@ -68,7 +68,7 @@ We currently support the functions: Supported function for operating over unwrapped ranges are: -- `rate(unwrapped-range)`: calculates per second rate of all values in the specified interval. +- `rate(unwrapped-range)`: calculates per second rate of the sum of all values in the specified interval. - `sum_over_time(unwrapped-range)`: the sum of all values in the specified interval. - `avg_over_time(unwrapped-range)`: the average value of all points in the specified interval. - `max_over_time(unwrapped-range)`: the maximum value of all points in the specified interval. diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 79a36a150fb88..689cb04ab7828 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -49,15 +49,36 @@ func TestEngine_LogsRateUnwrap(t *testing.T) { expected interface{} }{ { - `rate({app="foo"} | unwrap foo [30s])`, time.Unix(60, 0), logproto.FORWARD, 10, + `rate({app="foo"} | unwrap foo [30s])`, + time.Unix(60, 0), + logproto.FORWARD, + 10, + [][]logproto.Series{ + // 30s range the lower bound of the range is not inclusive only 15 samples will make it 60 included + {newSeries(testSize, offset(46, constantValue(1)), `{app="foo"}`)}, + }, + []SelectSampleParams{ + {&logproto.SampleQueryRequest{Start: time.Unix(30, 0), End: time.Unix(60, 0), Selector: `rate({app="foo"} | unwrap foo[30s])`}}, + }, + // SUM(n=47, 61, 1) = 15 + // 15 / 30 = 0.5 + promql.Vector{promql.Sample{Point: promql.Point{T: 60 * 1000, V: 0.5}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}}, + }, + { + `rate({app="foo"} | unwrap foo [30s])`, + time.Unix(60, 0), + logproto.FORWARD, + 10, [][]logproto.Series{ // 30s range the lower bound of the range is not inclusive only 15 samples will make it 60 included - {newSeries(testSize, offset(46, incValue(10)), `{app="foo"}`)}, + {newSeries(testSize, offset(46, incValue(1)), `{app="foo"}`)}, }, []SelectSampleParams{ {&logproto.SampleQueryRequest{Start: time.Unix(30, 0), End: time.Unix(60, 0), Selector: `rate({app="foo"} | unwrap foo[30s])`}}, }, - promql.Vector{promql.Sample{Point: promql.Point{T: 60 * 1000, V: 0.46666766666666665}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}}, + // SUM(n=47, 61, n) = 810 + // 810 / 30 = 27 + promql.Vector{promql.Sample{Point: promql.Point{T: 60 * 1000, V: 27}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}}, }, } { test := test @@ -150,7 +171,9 @@ func TestEngine_LogsInstantQuery(t *testing.T) { []SelectSampleParams{ {&logproto.SampleQueryRequest{Start: time.Unix(30, 0), End: time.Unix(60, 0), Selector: `rate({app="foo"} | unwrap foo[30s])`}}, }, - promql.Vector{promql.Sample{Point: promql.Point{T: 60 * 1000, V: 0.0}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}}, + // SUM(n=46, 61, 2) = 30 + // 30 / 30 = 1 + promql.Vector{promql.Sample{Point: promql.Point{T: 60 * 1000, V: 1.0}, Metric: labels.Labels{labels.Label{Name: "app", Value: "foo"}}}}, }, { `count_over_time({app="foo"} |~".+bar" [1m])`, time.Unix(60, 0), logproto.BACKWARD, 10, @@ -1287,7 +1310,10 @@ func TestEngine_RangeQuery(t *testing.T) { { `rate(({app=~"foo|bar"} |~".+bar" | unwrap bar)[1m])`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, 0, logproto.FORWARD, 100, [][]logproto.Series{ - {newSeries(testSize, factor(10, constantValue(2)), `{app="foo"}`), newSeries(testSize, factor(5, constantValue(2)), `{app="bar"}`)}, + { + newSeries(testSize, factor(10, constantValue(2)), `{app="foo"}`), + newSeries(testSize, factor(5, constantValue(2)), `{app="bar"}`), + }, }, []SelectSampleParams{ {&logproto.SampleQueryRequest{Start: time.Unix(0, 0), End: time.Unix(180, 0), Selector: `rate({app=~"foo|bar"}|~".+bar"|unwrap bar[1m])`}}, @@ -1295,11 +1321,11 @@ func TestEngine_RangeQuery(t *testing.T) { promql.Matrix{ promql.Series{ Metric: labels.Labels{{Name: "app", Value: "bar"}}, - Points: []promql.Point{{T: 60 * 1000, V: 0.0}, {T: 90 * 1000, V: 0.0}, {T: 120 * 1000, V: 0.0}, {T: 150 * 1000, V: 0.0}, {T: 180 * 1000, V: 0.0}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}}, }, promql.Series{ Metric: labels.Labels{{Name: "app", Value: "foo"}}, - Points: []promql.Point{{T: 60 * 1000, V: 0.0}, {T: 90 * 1000, V: 0.0}, {T: 120 * 1000, V: 0.0}, {T: 150 * 1000, V: 0.0}, {T: 180 * 1000, V: 0.0}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, }, }, }, diff --git a/pkg/logql/range_vector.go b/pkg/logql/range_vector.go index 713629a45ffcf..9b6f19d76cdae 100644 --- a/pkg/logql/range_vector.go +++ b/pkg/logql/range_vector.go @@ -218,13 +218,18 @@ func aggregator(r *syntax.RangeAggregationExpr) (RangeVectorAggregator, error) { } } -// rateLogs calculates the per-second rate of log lines. +// rateLogs calculates the per-second rate of log lines or values extracted +// from log lines func rateLogs(selRange time.Duration, computeValues bool) func(samples []promql.Point) float64 { return func(samples []promql.Point) float64 { if !computeValues { return float64(len(samples)) / selRange.Seconds() } - return extrapolatedRate(samples, selRange, true, true) + var result float64 + for _, sample := range samples { + result += sample.V + } + return result / selRange.Seconds() } }