From 2738f1c5e2d4015070f299050e917a3a8bcc6c02 Mon Sep 17 00:00:00 2001 From: Jimmiehan Date: Sun, 7 Nov 2021 17:33:12 +0800 Subject: [PATCH 1/6] Rule: Support ruleGroup limit (#4837) Signed-off-by: Jimmie Han --- CHANGELOG.md | 1 + pkg/rules/manager.go | 1 + pkg/rules/manager_test.go | 57 +++++++++++++- pkg/rules/rulespb/rpc.pb.go | 151 +++++++++++++++++++++--------------- pkg/rules/rulespb/rpc.proto | 3 +- 5 files changed, 150 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a45cc6df00..a017f01995 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#4847](https://github.com/thanos-io/thanos/pull/4847) Query: add `--alert.query-url` which is used in the user interface for rules/alerts pages. By default the HTTP listen address is used for this URL. - [#4856](https://github.com/thanos-io/thanos/pull/4856) Mixin: Add Query Frontend Grafana dashboard. - [#4874](https://github.com/thanos-io/thanos/pull/4874) Query: Add `--endpoint-strict` flag to statically configure Thanos API server endpoints. It is similar to `--store-strict` but supports passing any Thanos gRPC APIs: StoreAPI, MetadataAPI, RulesAPI, TargetsAPI and ExemplarsAPI. +- [#4868](https://github.com/thanos-io/thanos/pull/4868) Rule: Support ruleGroup limit introduced by Prometheus v2.31.0. ### Fixed diff --git a/pkg/rules/manager.go b/pkg/rules/manager.go index 9632af49c1..6ee198c3fa 100644 --- a/pkg/rules/manager.go +++ b/pkg/rules/manager.go @@ -44,6 +44,7 @@ func (g Group) toProto() *rulespb.RuleGroup { Name: g.Name(), File: g.OriginalFile, Interval: g.Interval().Seconds(), + Limit: int64(g.Limit()), PartialResponseStrategy: g.PartialResponseStrategy, // UTC needed due to https://github.com/gogo/protobuf/issues/519. LastEvaluation: g.GetLastEvaluation().UTC(), diff --git a/pkg/rules/manager_test.go b/pkg/rules/manager_test.go index 1a94fa8e9c..5fb8212a45 100644 --- a/pkg/rules/manager_test.go +++ b/pkg/rules/manager_test.go @@ -293,6 +293,7 @@ func TestConfigRuleAdapterUnmarshalMarshalYAML(t *testing.T) { - alert: some expr: up partial_response_strategy: ABORT + limit: 10 - name: something2 rules: - alert: some @@ -302,7 +303,8 @@ func TestConfigRuleAdapterUnmarshalMarshalYAML(t *testing.T) { b, err := yaml.Marshal(c) testutil.Ok(t, err) testutil.Equals(t, `groups: - - name: something1 + - limit: 10 + name: something1 rules: - alert: some expr: up @@ -397,3 +399,56 @@ groups: testutil.Ok(t, err) testutil.Equals(t, 0, len(thanosRuleMgr.RuleGroups())) } + +func TestManagerRunRulesWithRuleGroupLimit(t *testing.T) { + dir, err := ioutil.TempDir("", "test_rule_rule_groups") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() + filename := filepath.Join(dir, "with_limit.yaml") + testutil.Ok(t, ioutil.WriteFile(filename, []byte(` +groups: +- name: "something1" + limit: 1 + rules: + - alert: "some" + expr: "up>0" + for: 0s +`), os.ModePerm)) + + thanosRuleMgr := NewManager( + context.Background(), + nil, + dir, + rules.ManagerOptions{ + Logger: log.NewLogfmtLogger(os.Stderr), + Queryable: nopQueryable{}, + }, + func(partialResponseStrategy storepb.PartialResponseStrategy) rules.QueryFunc { + return func(ctx context.Context, q string, ts time.Time) (promql.Vector, error) { + return []promql.Sample{ + { + Point: promql.Point{T: 0, V: 1}, + Metric: labels.FromStrings("foo", "bar"), + }, + { + Point: promql.Point{T: 0, V: 1}, + Metric: labels.FromStrings("foo1", "bar1"), + }, + }, nil + } + }, + nil, + "http://localhost", + ) + thanosRuleMgr.Run() + defer func() { + thanosRuleMgr.Stop() + }() + testutil.Ok(t, thanosRuleMgr.Update(time.Millisecond, []string{filename})) + testutil.Equals(t, 1, len(thanosRuleMgr.protoRuleGroups())) + testutil.Equals(t, 1, len(thanosRuleMgr.protoRuleGroups()[0].Rules)) + testutil.Equals(t, string(rules.HealthUnknown), thanosRuleMgr.protoRuleGroups()[0].Rules[0].GetAlert().Health) + time.Sleep(time.Millisecond * 2) + testutil.Equals(t, string(rules.HealthBad), thanosRuleMgr.protoRuleGroups()[0].Rules[0].GetAlert().Health) + testutil.Equals(t, "exceeded limit of 1 with 2 alerts", thanosRuleMgr.protoRuleGroups()[0].Rules[0].GetAlert().LastError) +} diff --git a/pkg/rules/rulespb/rpc.pb.go b/pkg/rules/rulespb/rpc.pb.go index 12b21d7b84..4d85a902b8 100644 --- a/pkg/rules/rulespb/rpc.pb.go +++ b/pkg/rules/rulespb/rpc.pb.go @@ -279,6 +279,7 @@ type RuleGroup struct { Interval float64 `protobuf:"fixed64,4,opt,name=interval,proto3" json:"interval"` EvaluationDurationSeconds float64 `protobuf:"fixed64,5,opt,name=evaluation_duration_seconds,json=evaluationDurationSeconds,proto3" json:"evaluationTime"` LastEvaluation time.Time `protobuf:"bytes,6,opt,name=last_evaluation,json=lastEvaluation,proto3,stdtime" json:"lastEvaluation"` + Limit int64 `protobuf:"varint,9,opt,name=limit,proto3" json:"limit"` // Thanos specific. PartialResponseStrategy storepb.PartialResponseStrategy `protobuf:"varint,8,opt,name=PartialResponseStrategy,proto3,enum=thanos.PartialResponseStrategy" json:"partialResponseStrategy"` } @@ -551,69 +552,70 @@ func init() { func init() { proto.RegisterFile("rules/rulespb/rpc.proto", fileDescriptor_91b1d28f30eb5efb) } var fileDescriptor_91b1d28f30eb5efb = []byte{ - // 981 bytes of a gzipped FileDescriptorProto + // 999 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x56, 0xcd, 0x6e, 0xdb, 0x46, 0x10, 0x26, 0x2d, 0x91, 0x12, 0xc7, 0x76, 0xa2, 0x6e, 0x62, 0x98, 0x56, 0x0a, 0xd1, 0x10, 0x90, - 0xc2, 0x2d, 0x1a, 0xa9, 0xb0, 0x91, 0x14, 0x39, 0x15, 0x92, 0xad, 0xc6, 0x02, 0x0c, 0x37, 0x58, - 0x09, 0x3d, 0xa4, 0x07, 0x75, 0x25, 0x6f, 0x64, 0x01, 0x14, 0xc9, 0xec, 0xae, 0x5c, 0xe8, 0x01, - 0x7a, 0xcf, 0xb9, 0x2f, 0xd2, 0x7b, 0x4f, 0xbe, 0x14, 0xc8, 0xb1, 0x27, 0xb5, 0xb5, 0x6f, 0x7a, - 0x8a, 0x62, 0x77, 0x49, 0x51, 0x56, 0xe5, 0x3a, 0x69, 0xd5, 0x0b, 0x67, 0x77, 0xe6, 0x9b, 0xfd, - 0x99, 0xf9, 0xf8, 0x91, 0xb0, 0xcd, 0x46, 0x3e, 0xe5, 0x55, 0xf5, 0x8c, 0xba, 0x55, 0x16, 0xf5, - 0x2a, 0x11, 0x0b, 0x45, 0x88, 0x6c, 0x71, 0x4e, 0x82, 0x90, 0x17, 0x77, 0xb8, 0x08, 0x19, 0xad, - 0xaa, 0x67, 0xd4, 0xad, 0x8a, 0x71, 0x44, 0xb9, 0x86, 0x24, 0x21, 0x9f, 0x74, 0xa9, 0xbf, 0x10, - 0x7a, 0xd8, 0x0f, 0xfb, 0xa1, 0x1a, 0x56, 0xe5, 0x28, 0xf6, 0x7a, 0xfd, 0x30, 0xec, 0xfb, 0xb4, - 0xaa, 0x66, 0xdd, 0xd1, 0xeb, 0xaa, 0x18, 0x0c, 0x29, 0x17, 0x64, 0x18, 0x69, 0x40, 0xf9, 0x17, - 0x13, 0x36, 0xb0, 0x3c, 0x0a, 0xa6, 0x6f, 0x46, 0x94, 0x0b, 0xf4, 0x04, 0xb2, 0x72, 0x59, 0xd7, - 0xdc, 0x35, 0xf7, 0xee, 0xed, 0xef, 0x54, 0xf4, 0xa1, 0x2a, 0xf3, 0x98, 0x4a, 0x7b, 0x1c, 0x51, - 0xac, 0x60, 0xe8, 0x3b, 0xd8, 0x89, 0x08, 0x13, 0x03, 0xe2, 0x77, 0x18, 0xe5, 0x51, 0x18, 0x70, - 0xda, 0xe1, 0x82, 0x11, 0x41, 0xfb, 0x63, 0x77, 0x4d, 0xad, 0xe1, 0x25, 0x6b, 0xbc, 0xd4, 0x40, - 0x1c, 0xe3, 0x5a, 0x31, 0x0c, 0x6f, 0x47, 0xcb, 0x03, 0xe5, 0x4f, 0x20, 0x2b, 0xb7, 0x42, 0x39, - 0xc8, 0xd4, 0x4e, 0x4e, 0x0a, 0x06, 0x72, 0xc0, 0xaa, 0x9d, 0x34, 0x70, 0xbb, 0x60, 0x22, 0x00, - 0x1b, 0x37, 0x0e, 0xbf, 0xc1, 0x47, 0x85, 0xb5, 0xf2, 0xf7, 0xb0, 0x19, 0x9f, 0x4f, 0x2f, 0x80, - 0x3e, 0x05, 0xab, 0xcf, 0xc2, 0x51, 0xa4, 0x6e, 0xb1, 0xbe, 0xff, 0xd1, 0xfc, 0x2d, 0x5e, 0xc8, - 0xc0, 0xb1, 0x81, 0x35, 0x02, 0x15, 0x21, 0xf7, 0x03, 0x61, 0xc1, 0x20, 0xe8, 0xab, 0xe3, 0x3a, - 0xc7, 0x06, 0x4e, 0x1c, 0xf5, 0x3c, 0xd8, 0x8c, 0xf2, 0x91, 0x2f, 0xca, 0x87, 0x00, 0xb3, 0x5c, - 0x8e, 0x9e, 0x82, 0xad, 0x92, 0xb9, 0x6b, 0xee, 0x66, 0x96, 0xae, 0x5f, 0x87, 0xe9, 0xc4, 0x8b, - 0x41, 0x38, 0xb6, 0xe5, 0x5f, 0x33, 0xe0, 0xcc, 0x10, 0xe8, 0x63, 0xc8, 0x06, 0x64, 0xa8, 0x0b, - 0xed, 0xd4, 0xf3, 0xd3, 0x89, 0xa7, 0xe6, 0x58, 0x3d, 0x65, 0xf4, 0xf5, 0xc0, 0xa7, 0xfa, 0x4c, - 0x3a, 0x2a, 0xe7, 0x58, 0x3d, 0xd1, 0x13, 0xb0, 0x14, 0x7f, 0xdc, 0x8c, 0xda, 0x7f, 0x63, 0x7e, - 0xff, 0xba, 0x33, 0x9d, 0x78, 0x3a, 0x8c, 0xb5, 0x41, 0x7b, 0x90, 0x1f, 0x04, 0x82, 0xb2, 0x0b, - 0xe2, 0xbb, 0xd9, 0x5d, 0x73, 0xcf, 0xac, 0x6f, 0x4c, 0x27, 0xde, 0xcc, 0x87, 0x67, 0x23, 0x84, - 0xe1, 0x11, 0xbd, 0x20, 0xfe, 0x88, 0x88, 0x41, 0x18, 0x74, 0xce, 0x46, 0x4c, 0x0f, 0x38, 0xed, - 0x85, 0xc1, 0x19, 0x77, 0x2d, 0x95, 0x8c, 0xa6, 0x13, 0xef, 0x5e, 0x0a, 0x6b, 0x0f, 0x86, 0x14, - 0xef, 0xa4, 0xf3, 0xa3, 0x38, 0xab, 0xa5, 0x93, 0x50, 0x07, 0xee, 0xfb, 0x84, 0x8b, 0x4e, 0x8a, - 0x70, 0x6d, 0xd5, 0x96, 0x62, 0x45, 0xb3, 0xb3, 0x92, 0xb0, 0xb3, 0xd2, 0x4e, 0xd8, 0x59, 0x2f, - 0x5e, 0x4e, 0x3c, 0x43, 0xee, 0x23, 0x53, 0x1b, 0xb3, 0xcc, 0xb7, 0xbf, 0x7b, 0x26, 0x5e, 0xf0, - 0xa1, 0x0b, 0xd8, 0xbe, 0x85, 0x5a, 0x6e, 0xfe, 0xbd, 0x18, 0x58, 0x7f, 0x34, 0x9d, 0x78, 0xb7, - 0xb1, 0x10, 0xdf, 0xb6, 0x78, 0x39, 0x80, 0xac, 0x2c, 0x38, 0x7a, 0x0a, 0x0e, 0xa3, 0xbd, 0x90, - 0x9d, 0x49, 0x12, 0x69, 0xc6, 0x6d, 0xcd, 0x3a, 0x92, 0x04, 0x24, 0xf2, 0xd8, 0xc0, 0x29, 0x12, - 0x3d, 0x06, 0x8b, 0xf8, 0x94, 0x09, 0xd5, 0xe3, 0xf5, 0xfd, 0xcd, 0x24, 0xa5, 0x26, 0x9d, 0x92, - 0xa0, 0x2a, 0x3a, 0x47, 0xc2, 0x9f, 0x33, 0xb0, 0xa9, 0x82, 0xcd, 0x80, 0x0b, 0x12, 0xf4, 0x28, - 0x7a, 0x0e, 0xb6, 0xd2, 0x02, 0xbe, 0x48, 0xf4, 0x57, 0x27, 0xd2, 0xdd, 0xa2, 0xa2, 0x7e, 0x2f, - 0x2e, 0x64, 0x0c, 0xc4, 0xb1, 0x45, 0xc7, 0xb0, 0x4e, 0x82, 0x20, 0x14, 0xaa, 0x84, 0x3c, 0x3e, - 0xc3, 0x92, 0xfc, 0x07, 0x71, 0xfe, 0x3c, 0x1a, 0xcf, 0x4f, 0xd0, 0x01, 0x58, 0x5c, 0x10, 0x41, - 0xdd, 0x8c, 0x2a, 0x36, 0xba, 0x71, 0x8f, 0x96, 0x8c, 0x68, 0x4a, 0x2a, 0x10, 0xd6, 0x06, 0xb5, - 0xc0, 0x21, 0x3d, 0x31, 0xb8, 0xa0, 0x1d, 0x22, 0x14, 0x27, 0xef, 0xa0, 0xc3, 0x74, 0xe2, 0x21, - 0x9d, 0x50, 0x13, 0x9f, 0x87, 0xc3, 0x81, 0xa0, 0xc3, 0x48, 0x8c, 0x15, 0x1d, 0xf2, 0x89, 0x1f, - 0x79, 0x60, 0x49, 0x56, 0x50, 0xc5, 0x53, 0x47, 0xef, 0xaa, 0x1c, 0x58, 0x9b, 0x7f, 0x62, 0x8a, - 0xfd, 0x7f, 0x32, 0xe5, 0x47, 0x0b, 0x2c, 0x55, 0x8e, 0xb4, 0x58, 0xe6, 0x07, 0x14, 0x2b, 0x91, - 0x8a, 0xb5, 0xa5, 0x52, 0xe1, 0x81, 0xf5, 0x66, 0x44, 0xd9, 0x58, 0xd5, 0x3f, 0xbe, 0xb5, 0x72, - 0x60, 0x6d, 0xd0, 0x97, 0x50, 0xf8, 0xdb, 0x9b, 0x3c, 0x27, 0x03, 0x49, 0x0c, 0xdf, 0x3f, 0x5b, - 0x78, 0x73, 0x53, 0x7a, 0x59, 0xff, 0x91, 0x5e, 0xf6, 0xbf, 0xa7, 0xd7, 0x73, 0xb0, 0xd5, 0x8b, - 0xc0, 0xdd, 0x9c, 0x12, 0xbb, 0xad, 0x1b, 0x25, 0x4b, 0x5e, 0x05, 0x2d, 0xb8, 0x1a, 0x88, 0x63, - 0x8b, 0xca, 0x60, 0x9f, 0x53, 0xe2, 0x8b, 0x73, 0xa5, 0x03, 0x8e, 0xc6, 0x68, 0x0f, 0x8e, 0x2d, - 0x7a, 0x06, 0xa0, 0xd5, 0x89, 0xb1, 0x90, 0xb9, 0x8e, 0xc2, 0x6d, 0x4f, 0x27, 0xde, 0x03, 0x25, - 0x32, 0xd2, 0x99, 0xd2, 0x0d, 0x3b, 0x33, 0xe7, 0x5d, 0x4a, 0x09, 0x2b, 0x52, 0xca, 0xf5, 0x55, - 0x2a, 0x65, 0xf9, 0xa7, 0x0c, 0x6c, 0xde, 0x50, 0xa4, 0x3b, 0xbe, 0x42, 0x33, 0x6a, 0xad, 0xdd, - 0x42, 0xad, 0x94, 0x21, 0x99, 0x0f, 0x65, 0x48, 0xda, 0x9c, 0xec, 0x7b, 0x36, 0xc7, 0x5a, 0x55, - 0x73, 0xec, 0x15, 0x35, 0x27, 0xb7, 0xca, 0xe6, 0x7c, 0x76, 0x00, 0x90, 0xaa, 0x00, 0xda, 0x80, - 0x7c, 0xf3, 0xb4, 0x76, 0xd8, 0x6e, 0x7e, 0xdb, 0x28, 0x18, 0x68, 0x1d, 0x72, 0x2f, 0x1b, 0xa7, - 0x47, 0xcd, 0xd3, 0x17, 0xfa, 0xd7, 0xe7, 0xeb, 0x26, 0x96, 0xe3, 0xb5, 0xfd, 0xaf, 0xc0, 0x52, - 0xbf, 0x3e, 0xe8, 0x59, 0x32, 0x78, 0xb8, 0xec, 0x97, 0xad, 0xb8, 0xb5, 0xe0, 0xd5, 0x02, 0xf5, - 0x85, 0x59, 0x7f, 0x7c, 0xf9, 0x67, 0xc9, 0xb8, 0xbc, 0x2a, 0x99, 0xef, 0xae, 0x4a, 0xe6, 0x1f, - 0x57, 0x25, 0xf3, 0xed, 0x75, 0xc9, 0x78, 0x77, 0x5d, 0x32, 0x7e, 0xbb, 0x2e, 0x19, 0xaf, 0x72, - 0xf1, 0x6f, 0x6a, 0xd7, 0x56, 0x97, 0x3b, 0xf8, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x02, 0xf0, 0xe5, - 0x62, 0xbe, 0x0a, 0x00, 0x00, + 0xc2, 0x2d, 0x1a, 0xa9, 0xb0, 0x91, 0x14, 0x39, 0x15, 0xa2, 0xad, 0xc6, 0x02, 0x0c, 0x37, 0x58, + 0x09, 0x3d, 0xa4, 0x07, 0x95, 0x92, 0x37, 0x32, 0x01, 0x8a, 0x64, 0x76, 0x57, 0x2e, 0xf4, 0x00, + 0xbd, 0xe7, 0xdc, 0x17, 0xe9, 0xbd, 0x27, 0x1f, 0x73, 0xec, 0x49, 0x6d, 0xed, 0x9b, 0x0e, 0x7d, + 0x86, 0x62, 0x77, 0x49, 0x51, 0x56, 0xe5, 0x3a, 0x69, 0xd5, 0x8b, 0x66, 0x77, 0xe6, 0x9b, 0xfd, + 0x99, 0xf9, 0xf6, 0x13, 0x61, 0x9b, 0x8e, 0x02, 0xc2, 0xea, 0xf2, 0x37, 0xee, 0xd5, 0x69, 0xdc, + 0xaf, 0xc5, 0x34, 0xe2, 0x11, 0x32, 0xf9, 0xb9, 0x17, 0x46, 0xac, 0xbc, 0xc3, 0x78, 0x44, 0x49, + 0x5d, 0xfe, 0xc6, 0xbd, 0x3a, 0x1f, 0xc7, 0x84, 0x29, 0x48, 0x1a, 0x0a, 0xbc, 0x1e, 0x09, 0x16, + 0x42, 0x0f, 0x07, 0xd1, 0x20, 0x92, 0xc3, 0xba, 0x18, 0x25, 0x5e, 0x67, 0x10, 0x45, 0x83, 0x80, + 0xd4, 0xe5, 0xac, 0x37, 0x7a, 0x5d, 0xe7, 0xfe, 0x90, 0x30, 0xee, 0x0d, 0x63, 0x05, 0xa8, 0xfe, + 0xa2, 0xc3, 0x06, 0x16, 0x47, 0xc1, 0xe4, 0xcd, 0x88, 0x30, 0x8e, 0x9e, 0x40, 0x5e, 0x2c, 0x6b, + 0xeb, 0xbb, 0xfa, 0xde, 0xbd, 0xfd, 0x9d, 0x9a, 0x3a, 0x54, 0x6d, 0x1e, 0x53, 0xeb, 0x8c, 0x63, + 0x82, 0x25, 0x0c, 0x7d, 0x07, 0x3b, 0xb1, 0x47, 0xb9, 0xef, 0x05, 0x5d, 0x4a, 0x58, 0x1c, 0x85, + 0x8c, 0x74, 0x19, 0xa7, 0x1e, 0x27, 0x83, 0xb1, 0xbd, 0x26, 0xd7, 0x70, 0xd2, 0x35, 0x5e, 0x2a, + 0x20, 0x4e, 0x70, 0xed, 0x04, 0x86, 0xb7, 0xe3, 0xe5, 0x81, 0xea, 0x27, 0x90, 0x17, 0x5b, 0xa1, + 0x02, 0xe4, 0x1a, 0x27, 0x27, 0x25, 0x0d, 0x59, 0x60, 0x34, 0x4e, 0x9a, 0xb8, 0x53, 0xd2, 0x11, + 0x80, 0x89, 0x9b, 0x87, 0xdf, 0xe0, 0xa3, 0xd2, 0x5a, 0xf5, 0x7b, 0xd8, 0x4c, 0xce, 0xa7, 0x16, + 0x40, 0x9f, 0x82, 0x31, 0xa0, 0xd1, 0x28, 0x96, 0xb7, 0x58, 0xdf, 0xff, 0x68, 0xfe, 0x16, 0x2f, + 0x44, 0xe0, 0x58, 0xc3, 0x0a, 0x81, 0xca, 0x50, 0xf8, 0xc1, 0xa3, 0xa1, 0x1f, 0x0e, 0xe4, 0x71, + 0xad, 0x63, 0x0d, 0xa7, 0x0e, 0xb7, 0x08, 0x26, 0x25, 0x6c, 0x14, 0xf0, 0xea, 0x21, 0xc0, 0x2c, + 0x97, 0xa1, 0xa7, 0x60, 0xca, 0x64, 0x66, 0xeb, 0xbb, 0xb9, 0xa5, 0xeb, 0xbb, 0x30, 0x9d, 0x38, + 0x09, 0x08, 0x27, 0xb6, 0xfa, 0x67, 0x0e, 0xac, 0x19, 0x02, 0x7d, 0x0c, 0xf9, 0xd0, 0x1b, 0xaa, + 0x42, 0x5b, 0x6e, 0x71, 0x3a, 0x71, 0xe4, 0x1c, 0xcb, 0x5f, 0x11, 0x7d, 0xed, 0x07, 0x44, 0x9d, + 0x49, 0x45, 0xc5, 0x1c, 0xcb, 0x5f, 0xf4, 0x04, 0x0c, 0xc9, 0x1f, 0x3b, 0x27, 0xf7, 0xdf, 0x98, + 0xdf, 0xdf, 0xb5, 0xa6, 0x13, 0x47, 0x85, 0xb1, 0x32, 0x68, 0x0f, 0x8a, 0x7e, 0xc8, 0x09, 0xbd, + 0xf0, 0x02, 0x3b, 0xbf, 0xab, 0xef, 0xe9, 0xee, 0xc6, 0x74, 0xe2, 0xcc, 0x7c, 0x78, 0x36, 0x42, + 0x18, 0x1e, 0x91, 0x0b, 0x2f, 0x18, 0x79, 0xdc, 0x8f, 0xc2, 0xee, 0xd9, 0x88, 0xaa, 0x01, 0x23, + 0xfd, 0x28, 0x3c, 0x63, 0xb6, 0x21, 0x93, 0xd1, 0x74, 0xe2, 0xdc, 0xcb, 0x60, 0x1d, 0x7f, 0x48, + 0xf0, 0x4e, 0x36, 0x3f, 0x4a, 0xb2, 0xda, 0x2a, 0x09, 0x75, 0xe1, 0x7e, 0xe0, 0x31, 0xde, 0xcd, + 0x10, 0xb6, 0x29, 0xdb, 0x52, 0xae, 0x29, 0x76, 0xd6, 0x52, 0x76, 0xd6, 0x3a, 0x29, 0x3b, 0xdd, + 0xf2, 0xe5, 0xc4, 0xd1, 0xc4, 0x3e, 0x22, 0xb5, 0x39, 0xcb, 0x7c, 0xfb, 0x9b, 0xa3, 0xe3, 0x05, + 0x1f, 0x72, 0xc0, 0x08, 0xfc, 0xa1, 0xcf, 0x6d, 0x6b, 0x57, 0xdf, 0xcb, 0xa9, 0xfb, 0x4b, 0x07, + 0x56, 0x06, 0x5d, 0xc0, 0xf6, 0x2d, 0xdc, 0xb3, 0x8b, 0xef, 0x45, 0x51, 0xf7, 0xd1, 0x74, 0xe2, + 0xdc, 0x46, 0x53, 0x7c, 0xdb, 0xe2, 0xd5, 0x10, 0xf2, 0xa2, 0x23, 0xe8, 0x29, 0x58, 0x94, 0xf4, + 0x23, 0x7a, 0x26, 0x58, 0xa6, 0x28, 0xb9, 0x35, 0x6b, 0x59, 0x1a, 0x10, 0xc8, 0x63, 0x0d, 0x67, + 0x48, 0xf4, 0x18, 0x0c, 0x2f, 0x20, 0x94, 0x4b, 0x12, 0xac, 0xef, 0x6f, 0xa6, 0x29, 0x0d, 0xe1, + 0x14, 0x0c, 0x96, 0xd1, 0x39, 0x96, 0xfe, 0x9c, 0x83, 0x4d, 0x19, 0x6c, 0x85, 0x8c, 0x7b, 0x61, + 0x9f, 0xa0, 0xe7, 0x60, 0x4a, 0xb1, 0x60, 0x8b, 0x2f, 0xe1, 0xd5, 0x89, 0x70, 0xb7, 0x09, 0x77, + 0xef, 0x25, 0x95, 0x4e, 0x80, 0x38, 0xb1, 0xe8, 0x18, 0xd6, 0xbd, 0x30, 0x8c, 0xb8, 0xac, 0x31, + 0x4b, 0xce, 0xb0, 0x24, 0xff, 0x41, 0x92, 0x3f, 0x8f, 0xc6, 0xf3, 0x13, 0x74, 0x00, 0x06, 0xe3, + 0x1e, 0x27, 0x76, 0x4e, 0x16, 0x1b, 0xdd, 0xb8, 0x47, 0x5b, 0x44, 0x54, 0xcf, 0x24, 0x08, 0x2b, + 0x83, 0xda, 0x60, 0x79, 0x7d, 0xee, 0x5f, 0x90, 0xae, 0xc7, 0x25, 0x69, 0xef, 0xe0, 0xcb, 0x74, + 0xe2, 0x20, 0x95, 0xd0, 0xe0, 0x9f, 0x47, 0x43, 0x9f, 0x93, 0x61, 0xcc, 0xc7, 0x92, 0x2f, 0xc5, + 0xd4, 0x2f, 0x98, 0x22, 0x68, 0x43, 0x24, 0x91, 0x2d, 0xb5, 0xab, 0x74, 0x60, 0x65, 0xfe, 0x89, + 0x29, 0xe6, 0xff, 0xc9, 0x94, 0x1f, 0x0d, 0x30, 0x64, 0x39, 0xb2, 0x62, 0xe9, 0x1f, 0x50, 0xac, + 0x54, 0x4b, 0xd6, 0x96, 0x6a, 0x89, 0x03, 0xc6, 0x9b, 0x11, 0xa1, 0x63, 0x59, 0xff, 0xe4, 0xd6, + 0xd2, 0x81, 0x95, 0x41, 0x5f, 0x42, 0xe9, 0x6f, 0x4f, 0x7d, 0x4e, 0x27, 0xd2, 0x18, 0xbe, 0x7f, + 0xb6, 0xf0, 0xb4, 0x33, 0x7a, 0x19, 0xff, 0x91, 0x5e, 0xe6, 0xbf, 0xa7, 0xd7, 0x73, 0x30, 0xe5, + 0x43, 0x60, 0x76, 0x41, 0xaa, 0xe1, 0xd6, 0x8d, 0x92, 0xa5, 0x4f, 0x41, 0x29, 0xb2, 0x02, 0xe2, + 0xc4, 0xa2, 0x2a, 0x98, 0xe7, 0xc4, 0x0b, 0xf8, 0xb9, 0xd4, 0x01, 0x4b, 0x61, 0x94, 0x07, 0x27, + 0x16, 0x3d, 0x03, 0x50, 0xf2, 0x45, 0x69, 0x44, 0xa5, 0xc4, 0x58, 0xee, 0xf6, 0x74, 0xe2, 0x3c, + 0x90, 0x2a, 0x24, 0x9c, 0x19, 0xdd, 0xb0, 0x35, 0x73, 0xde, 0x25, 0xa5, 0xb0, 0x22, 0x29, 0x5d, + 0x5f, 0xa5, 0x94, 0x56, 0x7f, 0xca, 0xc1, 0xe6, 0x0d, 0x45, 0xba, 0xe3, 0x6f, 0x6a, 0x46, 0xad, + 0xb5, 0x5b, 0xa8, 0x95, 0x31, 0x24, 0xf7, 0xa1, 0x0c, 0xc9, 0x9a, 0x93, 0x7f, 0xcf, 0xe6, 0x18, + 0xab, 0x6a, 0x8e, 0xb9, 0xa2, 0xe6, 0x14, 0x56, 0xd9, 0x9c, 0xcf, 0x0e, 0x00, 0x32, 0x15, 0x40, + 0x1b, 0x50, 0x6c, 0x9d, 0x36, 0x0e, 0x3b, 0xad, 0x6f, 0x9b, 0x25, 0x0d, 0xad, 0x43, 0xe1, 0x65, + 0xf3, 0xf4, 0xa8, 0x75, 0xfa, 0x42, 0x7d, 0x1b, 0x7d, 0xdd, 0xc2, 0x62, 0xbc, 0xb6, 0xff, 0x15, + 0x18, 0xf2, 0xdb, 0x08, 0x3d, 0x4b, 0x07, 0x0f, 0x97, 0x7d, 0xd3, 0x95, 0xb7, 0x16, 0xbc, 0x4a, + 0xa0, 0xbe, 0xd0, 0xdd, 0xc7, 0x97, 0x7f, 0x54, 0xb4, 0xcb, 0xab, 0x8a, 0xfe, 0xee, 0xaa, 0xa2, + 0xff, 0x7e, 0x55, 0xd1, 0xdf, 0x5e, 0x57, 0xb4, 0x77, 0xd7, 0x15, 0xed, 0xd7, 0xeb, 0x8a, 0xf6, + 0xaa, 0x90, 0x7c, 0xc7, 0xf6, 0x4c, 0x79, 0xb9, 0x83, 0xbf, 0x02, 0x00, 0x00, 0xff, 0xff, 0x8f, + 0x24, 0x78, 0x7a, 0xdf, 0x0a, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -884,6 +886,11 @@ func (m *RuleGroup) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Limit != 0 { + i = encodeVarintRpc(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x48 + } if m.PartialResponseStrategy != 0 { i = encodeVarintRpc(dAtA, i, uint64(m.PartialResponseStrategy)) i-- @@ -1375,6 +1382,9 @@ func (m *RuleGroup) Size() (n int) { if m.PartialResponseStrategy != 0 { n += 1 + sovRpc(uint64(m.PartialResponseStrategy)) } + if m.Limit != 0 { + n += 1 + sovRpc(uint64(m.Limit)) + } return n } @@ -2015,6 +2025,25 @@ func (m *RuleGroup) Unmarshal(dAtA []byte) error { break } } + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) + } + m.Limit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Limit |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) diff --git a/pkg/rules/rulespb/rpc.proto b/pkg/rules/rulespb/rpc.proto index 2d791f6cf0..e95d3a2ee4 100644 --- a/pkg/rules/rulespb/rpc.proto +++ b/pkg/rules/rulespb/rpc.proto @@ -71,7 +71,8 @@ message RuleGroup { repeated Rule rules = 3 [(gogoproto.jsontag) = "rules" ]; double interval = 4 [(gogoproto.jsontag) = "interval" ]; double evaluation_duration_seconds = 5 [(gogoproto.jsontag) = "evaluationTime" ]; // TODO: Is it really second? - google.protobuf.Timestamp last_evaluation = 6 [(gogoproto.jsontag) = "lastEvaluation", (gogoproto.stdtime) = true, (gogoproto.nullable) = false ]; + google.protobuf.Timestamp last_evaluation = 6 [(gogoproto.jsontag) = "lastEvaluation", (gogoproto.stdtime) = true, (gogoproto.nullable) = false]; + int64 limit = 9 [(gogoproto.jsontag) = "limit"]; // Thanos specific. PartialResponseStrategy PartialResponseStrategy = 8 [(gogoproto.jsontag) = "partialResponseStrategy" ]; From c0a35317e314d15f78bc0aed2128bf343709d576 Mon Sep 17 00:00:00 2001 From: Jimmie Han Date: Sun, 14 Nov 2021 17:10:04 +0800 Subject: [PATCH 2/6] Rules: format rpc.proto Signed-off-by: Jimmie Han --- pkg/rules/rulespb/rpc.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/rules/rulespb/rpc.proto b/pkg/rules/rulespb/rpc.proto index e95d3a2ee4..2049334610 100644 --- a/pkg/rules/rulespb/rpc.proto +++ b/pkg/rules/rulespb/rpc.proto @@ -71,8 +71,8 @@ message RuleGroup { repeated Rule rules = 3 [(gogoproto.jsontag) = "rules" ]; double interval = 4 [(gogoproto.jsontag) = "interval" ]; double evaluation_duration_seconds = 5 [(gogoproto.jsontag) = "evaluationTime" ]; // TODO: Is it really second? - google.protobuf.Timestamp last_evaluation = 6 [(gogoproto.jsontag) = "lastEvaluation", (gogoproto.stdtime) = true, (gogoproto.nullable) = false]; - int64 limit = 9 [(gogoproto.jsontag) = "limit"]; + google.protobuf.Timestamp last_evaluation = 6 [(gogoproto.jsontag) = "lastEvaluation", (gogoproto.stdtime) = true, (gogoproto.nullable) = false ]; + int64 limit = 9 [(gogoproto.jsontag) = "limit" ]; // Thanos specific. PartialResponseStrategy PartialResponseStrategy = 8 [(gogoproto.jsontag) = "partialResponseStrategy" ]; From a42d4c4378185b93a3b3871a881d8cfd2c0bd240 Mon Sep 17 00:00:00 2001 From: Jimmie Han Date: Sun, 14 Nov 2021 17:48:59 +0800 Subject: [PATCH 3/6] Rules: fix unit test Signed-off-by: Jimmie Han --- pkg/rules/rulespb/custom_test.go | 5 +++-- pkg/testutil/testpromcompatibility/api_compatibility.go | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/rules/rulespb/custom_test.go b/pkg/rules/rulespb/custom_test.go index eacbd4df3c..836075f4df 100644 --- a/pkg/rules/rulespb/custom_test.go +++ b/pkg/rules/rulespb/custom_test.go @@ -57,6 +57,7 @@ func TestJSONUnmarshalMarshal(t *testing.T) { Interval: 2442, LastEvaluation: now, EvaluationDurationSeconds: 2.1, + Limit: 0, PartialResponseStrategy: storepb.PartialResponseStrategy_ABORT, Rules: []*Rule{}, }, @@ -78,7 +79,7 @@ func TestJSONUnmarshalMarshal(t *testing.T) { }, }, // Different than input due to default enum fields. - expectedJSONOutput: `{"groups":[{"name":"","file":"","rules":[],"interval":0,"evaluationTime":0,"lastEvaluation":"0001-01-01T00:00:00Z","partialResponseStrategy":"ABORT"}]}`, + expectedJSONOutput: `{"groups":[{"name":"","file":"","rules":[],"interval":0,"evaluationTime":0,"lastEvaluation":"0001-01-01T00:00:00Z","limit":0,"partialResponseStrategy":"ABORT"}]}`, }, { name: "one valid group, with 1 with no rule type", @@ -230,7 +231,7 @@ func TestJSONUnmarshalMarshal(t *testing.T) { }, }, // Different than input due to the alerts slice being initialized to a zero-length slice instead of nil. - expectedJSONOutput: `{"groups":[{"name":"group1","file":"file1.yml","rules":[{"state":"pending","name":"alert1","query":"up == 0","duration":60,"labels":{"a2":"b2","c2":"d2"},"annotations":{"ann1":"ann44","ann2":"ann33"},"alerts":[],"health":"health2","lastError":"1","evaluationTime":1.1,"lastEvaluation":"0001-01-01T00:00:00Z","type":"alerting"}],"interval":2442,"evaluationTime":2.1,"lastEvaluation":"0001-01-01T00:00:00Z","partialResponseStrategy":"ABORT"}]}`, + expectedJSONOutput: `{"groups":[{"name":"group1","file":"file1.yml","rules":[{"state":"pending","name":"alert1","query":"up == 0","duration":60,"labels":{"a2":"b2","c2":"d2"},"annotations":{"ann1":"ann44","ann2":"ann33"},"alerts":[],"health":"health2","lastError":"1","evaluationTime":1.1,"lastEvaluation":"0001-01-01T00:00:00Z","type":"alerting"}],"interval":2442,"evaluationTime":2.1,"lastEvaluation":"0001-01-01T00:00:00Z","limit":0,"partialResponseStrategy":"ABORT"}]}`, }, { name: "one valid group, with 1 rule and alert each and second empty group.", diff --git a/pkg/testutil/testpromcompatibility/api_compatibility.go b/pkg/testutil/testpromcompatibility/api_compatibility.go index ecf357f8bb..5dde3b53b0 100644 --- a/pkg/testutil/testpromcompatibility/api_compatibility.go +++ b/pkg/testutil/testpromcompatibility/api_compatibility.go @@ -24,6 +24,7 @@ type RuleGroup struct { Interval float64 `json:"interval"` EvaluationTime float64 `json:"evaluationTime"` LastEvaluation time.Time `json:"lastEvaluation"` + Limit int `json:"limit"` PartialResponseStrategy string `json:"partialResponseStrategy"` } From a79983d5bae1adba4dabad4ebb8de5f298846afc Mon Sep 17 00:00:00 2001 From: Jimmie Han Date: Tue, 16 Nov 2021 21:06:35 +0800 Subject: [PATCH 4/6] Rules: add e2e test Signed-off-by: Jimmie Han --- test/e2e/rule_test.go | 15 +++++++++++++++ test/e2e/rules_api_test.go | 18 +++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index dd2d6a927b..d8d4826bac 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -98,6 +98,21 @@ groups: annotations: summary: "I always complain and I have been loaded via sighup signal." ` + testAlertRuleWithLimit = ` +groups: +- name: example_with_limit + interval: 1s + limit: 1 + partial_response_strategy: "WARN" + rules: + - alert: TestAlert_WithLimit + expr: up + labels: + severity: page + annotations: + summary: "with limit" +` + testRuleRecordAbsentMetric = ` groups: - name: example_record_rules diff --git a/test/e2e/rules_api_test.go b/test/e2e/rules_api_test.go index 75a088e08b..7187bf48f7 100644 --- a/test/e2e/rules_api_test.go +++ b/test/e2e/rules_api_test.go @@ -42,7 +42,8 @@ func TestRulesAPI_Fanout(t *testing.T) { thanosRulesSubDir := filepath.Join("thanos-rules") testutil.Ok(t, os.MkdirAll(filepath.Join(e.SharedDir(), thanosRulesSubDir), os.ModePerm)) createRuleFiles(t, filepath.Join(e.SharedDir(), thanosRulesSubDir)) - + // We create a rule group with limit. + createRuleFile(t, filepath.Join(e.SharedDir(), thanosRulesSubDir, "rules-with-limit.yaml"), testAlertRuleWithLimit) // 2x Prometheus. prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar( e, @@ -135,6 +136,21 @@ func TestRulesAPI_Fanout(t *testing.T) { }), }, }, + { + Name: "example_with_limit", + File: "/shared/thanos-rules/rules-with-limit.yaml", + Limit: 1, + Rules: []*rulespb.Rule{ + rulespb.NewAlertingRule(&rulespb.Alert{ + Name: "TestAlert_WithLimit", + State: rulespb.AlertState_FIRING, + Query: "up", + Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ + {Name: "severity", Value: "page"}, + }}, + }), + }, + }, }) } From 7b1ee8dedc088d89f0713137b950f5f66dd3bb1b Mon Sep 17 00:00:00 2001 From: Jimmie Han Date: Thu, 18 Nov 2021 13:05:34 +0800 Subject: [PATCH 5/6] Rules: add e2e rule test case hit limit Signed-off-by: Jimmie Han --- test/e2e/query_test.go | 2 ++ test/e2e/rule_test.go | 4 ++-- test/e2e/rules_api_test.go | 11 +++++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 07ed2686b4..e93996ea4d 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" + "github.com/prometheus/prometheus/rules" "github.com/thanos-io/thanos/pkg/metadata/metadatapb" "github.com/thanos-io/thanos/pkg/rules/rulespb" "github.com/thanos-io/thanos/pkg/store/labelpb" @@ -563,6 +564,7 @@ config: {Name: "prometheus", Value: "p1"}, {Name: "severity", Value: "page"}, }}, + Health: string(rules.HealthGood), }), }, }, diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index d8d4826bac..8363b3b398 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -102,11 +102,11 @@ groups: groups: - name: example_with_limit interval: 1s - limit: 1 partial_response_strategy: "WARN" + limit: 1 rules: - alert: TestAlert_WithLimit - expr: up + expr: 'promhttp_metric_handler_requests_total' # It has more than one labels. labels: severity: page annotations: diff --git a/test/e2e/rules_api_test.go b/test/e2e/rules_api_test.go index 7187bf48f7..dfb8ec35aa 100644 --- a/test/e2e/rules_api_test.go +++ b/test/e2e/rules_api_test.go @@ -16,6 +16,7 @@ import ( "github.com/efficientgo/e2e" "github.com/go-kit/kit/log" "github.com/pkg/errors" + "github.com/prometheus/prometheus/rules" "github.com/thanos-io/thanos/pkg/httpconfig" "github.com/thanos-io/thanos/pkg/promclient" @@ -105,6 +106,7 @@ func TestRulesAPI_Fanout(t *testing.T) { {Name: "prometheus", Value: "ha"}, {Name: "severity", Value: "page"}, }}, + Health: string(rules.HealthGood), }), }, }, @@ -119,6 +121,7 @@ func TestRulesAPI_Fanout(t *testing.T) { Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ {Name: "severity", Value: "page"}, }}, + Health: string(rules.HealthGood), }), }, }, @@ -133,6 +136,7 @@ func TestRulesAPI_Fanout(t *testing.T) { Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ {Name: "severity", Value: "page"}, }}, + Health: string(rules.HealthGood), }), }, }, @@ -143,11 +147,12 @@ func TestRulesAPI_Fanout(t *testing.T) { Rules: []*rulespb.Rule{ rulespb.NewAlertingRule(&rulespb.Alert{ Name: "TestAlert_WithLimit", - State: rulespb.AlertState_FIRING, - Query: "up", + State: rulespb.AlertState_INACTIVE, + Query: `promhttp_metric_handler_requests_total`, Labels: labelpb.ZLabelSet{Labels: []labelpb.ZLabel{ {Name: "severity", Value: "page"}, }}, + Health: string(rules.HealthBad), }), }, }, @@ -191,12 +196,14 @@ func ruleAndAssert(t *testing.T, ctx context.Context, addr, typ string, want []* State: alert.State, Query: alert.Query, Labels: alert.Labels, + Health: alert.Health, }) } else if rec := r.GetRecording(); rec != nil { res[ig].Rules[ir] = rulespb.NewAlertingRule(&rulespb.Alert{ Name: rec.Name, Query: rec.Query, Labels: rec.Labels, + Health: rec.Health, }) } } From 2de2981110f30a5f51dedcd08914bb200912f1ce Mon Sep 17 00:00:00 2001 From: Jimmie Han Date: Thu, 18 Nov 2021 14:40:12 +0800 Subject: [PATCH 6/6] fix unit test ci, use runutil.Retry instead of time.Sleep. Signed-off-by: Jimmie Han --- pkg/rules/manager_test.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/rules/manager_test.go b/pkg/rules/manager_test.go index 5fb8212a45..0e5142a14e 100644 --- a/pkg/rules/manager_test.go +++ b/pkg/rules/manager_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/go-kit/kit/log" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/exemplar" "github.com/prometheus/prometheus/pkg/labels" @@ -23,6 +24,7 @@ import ( "github.com/prometheus/prometheus/rules" "github.com/prometheus/prometheus/storage" "github.com/thanos-io/thanos/pkg/extprom" + "github.com/thanos-io/thanos/pkg/runutil" "gopkg.in/yaml.v3" "github.com/thanos-io/thanos/pkg/store/storepb" @@ -385,9 +387,7 @@ groups: // We need to run the underlying rule managers to update them more than // once (otherwise there's a deadlock). thanosRuleMgr.Run() - defer func() { - thanosRuleMgr.Stop() - }() + t.Cleanup(thanosRuleMgr.Stop) err = thanosRuleMgr.Update(1*time.Second, []string{ filepath.Join(dir, "no_strategy.yaml"), @@ -403,11 +403,12 @@ groups: func TestManagerRunRulesWithRuleGroupLimit(t *testing.T) { dir, err := ioutil.TempDir("", "test_rule_rule_groups") testutil.Ok(t, err) - defer func() { testutil.Ok(t, os.RemoveAll(dir)) }() + t.Cleanup(func() { testutil.Ok(t, os.RemoveAll(dir)) }) filename := filepath.Join(dir, "with_limit.yaml") testutil.Ok(t, ioutil.WriteFile(filename, []byte(` groups: - name: "something1" + interval: 1ms limit: 1 rules: - alert: "some" @@ -441,14 +442,17 @@ groups: "http://localhost", ) thanosRuleMgr.Run() - defer func() { - thanosRuleMgr.Stop() - }() + t.Cleanup(thanosRuleMgr.Stop) testutil.Ok(t, thanosRuleMgr.Update(time.Millisecond, []string{filename})) testutil.Equals(t, 1, len(thanosRuleMgr.protoRuleGroups())) testutil.Equals(t, 1, len(thanosRuleMgr.protoRuleGroups()[0].Rules)) - testutil.Equals(t, string(rules.HealthUnknown), thanosRuleMgr.protoRuleGroups()[0].Rules[0].GetAlert().Health) - time.Sleep(time.Millisecond * 2) - testutil.Equals(t, string(rules.HealthBad), thanosRuleMgr.protoRuleGroups()[0].Rules[0].GetAlert().Health) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + testutil.Ok(t, runutil.Retry(time.Millisecond, ctx.Done(), func() error { + if thanosRuleMgr.protoRuleGroups()[0].Rules[0].GetAlert().Health != string(rules.HealthBad) { + return errors.New("expect HealthBad") + } + return nil + })) testutil.Equals(t, "exceeded limit of 1 with 2 alerts", thanosRuleMgr.protoRuleGroups()[0].Rules[0].GetAlert().LastError) }