From eca8961df8bab3ccb639da3136f38b0ee75680b1 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Thu, 28 Apr 2022 11:12:15 +0200 Subject: [PATCH] Fix mimir-continuous-test when changing configured num-series Signed-off-by: Marco Pracucci --- CHANGELOG.md | 2 +- pkg/continuoustest/write_read_series.go | 26 +++++++++++--------- pkg/continuoustest/write_read_series_test.go | 20 +++++++-------- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index baf484088c0..0b85d0f5742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,7 +95,7 @@ ### Tools * [FEATURE] Added a `markblocks` tool that creates `no-compact` and `delete` marks for the blocks. #1551 -* [FEATURE] Added `mimir-continuous-test` tool to continuously run smoke tests on live Mimir clusters. #1535 #1540 #1653 #1603 #1630 #1691 #1675 #1676 #1692 #1706 #1709 #1777 #1778 +* [FEATURE] Added `mimir-continuous-test` tool to continuously run smoke tests on live Mimir clusters. #1535 #1540 #1653 #1603 #1630 #1691 #1675 #1676 #1692 #1706 #1709 #1775 #1777 #1778 * [FEATURE] Added `mimir-rules-action` GitHub action, located at `operations/mimir-rules-action/`, used to lint, prepare, verify, diff, and sync rules to a Mimir cluster. #1723 ## 2.0.0 diff --git a/pkg/continuoustest/write_read_series.go b/pkg/continuoustest/write_read_series.go index 8a55eb0d4bb..97a2d740dc5 100644 --- a/pkg/continuoustest/write_read_series.go +++ b/pkg/continuoustest/write_read_series.go @@ -22,6 +22,14 @@ const ( metricName = "mimir_continuous_test_sine_wave" ) +var ( + // We use max_over_time() with a 1s range selector in order to fetch only the samples we previously + // wrote and ensure the PromQL lookback period doesn't influence query results. This help to avoid + // false positives when finding the last written sample, or when restarting the testing tool with + // a different number of configured series to write and read. + queryMetricSum = fmt.Sprintf("sum(max_over_time(%s[1s]))", metricName) +) + type WriteReadSeriesTestConfig struct { NumSeries int MaxQueryAge time.Duration @@ -202,13 +210,12 @@ func (t *WriteReadSeriesTest) runRangeQueryAndVerifyResult(ctx context.Context, } step := getQueryStep(start, end, writeInterval) - query := fmt.Sprintf("sum(%s)", metricName) - logger := log.With(t.logger, "query", query, "start", start.UnixMilli(), "end", end.UnixMilli(), "step", step, "results_cache", strconv.FormatBool(resultsCacheEnabled)) + logger := log.With(t.logger, "query", queryMetricSum, "start", start.UnixMilli(), "end", end.UnixMilli(), "step", step, "results_cache", strconv.FormatBool(resultsCacheEnabled)) level.Debug(logger).Log("msg", "Running range query") t.metrics.queriesTotal.Inc() - matrix, err := t.client.QueryRange(ctx, query, start, end, step, WithResultsCacheEnabled(resultsCacheEnabled)) + matrix, err := t.client.QueryRange(ctx, queryMetricSum, start, end, step, WithResultsCacheEnabled(resultsCacheEnabled)) if err != nil { t.metrics.queriesFailedTotal.Inc() level.Warn(logger).Log("msg", "Failed to execute range query", "err", err) @@ -232,13 +239,11 @@ func (t *WriteReadSeriesTest) runInstantQueryAndVerifyResult(ctx context.Context return } - query := fmt.Sprintf("sum(%s)", metricName) - - logger := log.With(t.logger, "query", query, "ts", ts.UnixMilli(), "results_cache", strconv.FormatBool(resultsCacheEnabled)) + logger := log.With(t.logger, "query", queryMetricSum, "ts", ts.UnixMilli(), "results_cache", strconv.FormatBool(resultsCacheEnabled)) level.Debug(logger).Log("msg", "Running instant query") t.metrics.queriesTotal.Inc() - vector, err := t.client.Query(ctx, query, ts, WithResultsCacheEnabled(resultsCacheEnabled)) + vector, err := t.client.Query(ctx, queryMetricSum, ts, WithResultsCacheEnabled(resultsCacheEnabled)) if err != nil { t.metrics.queriesFailedTotal.Inc() level.Warn(logger).Log("msg", "Failed to execute instant query", "err", err) @@ -275,9 +280,6 @@ func (t *WriteReadSeriesTest) nextWriteTimestamp(now time.Time) time.Time { } func (t *WriteReadSeriesTest) findPreviouslyWrittenTimeRange(ctx context.Context, now time.Time) (from, to time.Time) { - // We use max_over_time() with a 1s range selector in order to fetch only the samples we previously - // wrote and ensure the PromQL lookback period doesn't influence query results. - query := fmt.Sprintf("sum(max_over_time(%s[1s]))", metricName) end := alignTimestampToInterval(now, writeInterval) step := writeInterval @@ -290,10 +292,10 @@ func (t *WriteReadSeriesTest) findPreviouslyWrittenTimeRange(ctx context.Context return } - logger := log.With(t.logger, "query", query, "start", start, "end", end, "step", step) + logger := log.With(t.logger, "query", queryMetricSum, "start", start, "end", end, "step", step) level.Debug(logger).Log("msg", "Executing query to find previously written samples") - matrix, err := t.client.QueryRange(ctx, query, start, end, step, WithResultsCacheEnabled(false)) + matrix, err := t.client.QueryRange(ctx, queryMetricSum, start, end, step, WithResultsCacheEnabled(false)) if err != nil { level.Warn(logger).Log("msg", "Failed to execute range query used to find previously written samples", "err", err) return diff --git a/pkg/continuoustest/write_read_series_test.go b/pkg/continuoustest/write_read_series_test.go index 262a9604538..51632f0d49b 100644 --- a/pkg/continuoustest/write_read_series_test.go +++ b/pkg/continuoustest/write_read_series_test.go @@ -42,10 +42,10 @@ func TestWriteReadSeriesTest_Run(t *testing.T) { assert.Equal(t, int64(1000), test.lastWrittenTimestamp.Unix()) client.AssertNumberOfCalls(t, "QueryRange", 4) - client.AssertCalled(t, "QueryRange", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything) + client.AssertCalled(t, "QueryRange", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything) client.AssertNumberOfCalls(t, "Query", 4) - client.AssertCalled(t, "Query", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), mock.Anything) + client.AssertCalled(t, "Query", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), mock.Anything) assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP mimir_continuous_test_writes_total Total number of attempted write requests. @@ -81,10 +81,10 @@ func TestWriteReadSeriesTest_Run(t *testing.T) { assert.Equal(t, int64(980), test.lastWrittenTimestamp.Unix()) client.AssertNumberOfCalls(t, "QueryRange", 4) - client.AssertCalled(t, "QueryRange", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(980, 0), time.Unix(980, 0), writeInterval, mock.Anything) + client.AssertCalled(t, "QueryRange", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(980, 0), time.Unix(980, 0), writeInterval, mock.Anything) client.AssertNumberOfCalls(t, "Query", 4) - client.AssertCalled(t, "Query", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(980, 0), mock.Anything) + client.AssertCalled(t, "Query", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(980, 0), mock.Anything) assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP mimir_continuous_test_writes_total Total number of attempted write requests. @@ -123,10 +123,10 @@ func TestWriteReadSeriesTest_Run(t *testing.T) { assert.Equal(t, int64(1000), test.lastWrittenTimestamp.Unix()) client.AssertNumberOfCalls(t, "QueryRange", 4) - client.AssertCalled(t, "QueryRange", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(960, 0), time.Unix(1000, 0), writeInterval, mock.Anything) + client.AssertCalled(t, "QueryRange", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(960, 0), time.Unix(1000, 0), writeInterval, mock.Anything) client.AssertNumberOfCalls(t, "Query", 4) - client.AssertCalled(t, "Query", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), mock.Anything) + client.AssertCalled(t, "Query", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), mock.Anything) assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP mimir_continuous_test_writes_total Total number of attempted write requests. @@ -259,10 +259,10 @@ func TestWriteReadSeriesTest_Run(t *testing.T) { assert.Equal(t, int64(1000), test.lastWrittenTimestamp.Unix()) client.AssertNumberOfCalls(t, "QueryRange", 4) - client.AssertCalled(t, "QueryRange", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything) + client.AssertCalled(t, "QueryRange", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything) client.AssertNumberOfCalls(t, "Query", 4) - client.AssertCalled(t, "Query", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), mock.Anything) + client.AssertCalled(t, "Query", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), mock.Anything) assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP mimir_continuous_test_writes_total Total number of attempted write requests. @@ -313,10 +313,10 @@ func TestWriteReadSeriesTest_Run(t *testing.T) { assert.Equal(t, int64(1000), test.lastWrittenTimestamp.Unix()) client.AssertNumberOfCalls(t, "QueryRange", 4) - client.AssertCalled(t, "QueryRange", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything) + client.AssertCalled(t, "QueryRange", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), time.Unix(1000, 0), writeInterval, mock.Anything) client.AssertNumberOfCalls(t, "Query", 4) - client.AssertCalled(t, "Query", mock.Anything, "sum(mimir_continuous_test_sine_wave)", time.Unix(1000, 0), mock.Anything) + client.AssertCalled(t, "Query", mock.Anything, "sum(max_over_time(mimir_continuous_test_sine_wave[1s]))", time.Unix(1000, 0), mock.Anything) assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP mimir_continuous_test_writes_total Total number of attempted write requests.