From fc12ef4a9fa9cb93eef2dc85e6fc3fe26384a4f9 Mon Sep 17 00:00:00 2001 From: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com> Date: Sun, 8 Mar 2020 22:30:54 +0000 Subject: [PATCH 1/5] Add test for alert queue Pop after multiple Push Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com> --- pkg/alert/alert_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/alert/alert_test.go b/pkg/alert/alert_test.go index 7139dfc652..d75265866b 100644 --- a/pkg/alert/alert_test.go +++ b/pkg/alert/alert_test.go @@ -19,6 +19,30 @@ import ( "github.com/thanos-io/thanos/pkg/testutil" ) +func TestQueue_Pop_all_Pushed(t *testing.T) { + qcapacity := 5 + batchsize := 1 + pushes := 3 + + q := NewQueue( + nil, nil, qcapacity, batchsize, nil, nil, + ) + for i := 0; i < pushes; i++ { + q.Push([]*Alert{ + {}, + {}, + }) + } + + timeoutc := time.After(time.Second) + popped := 0 + for p := q.Pop(timeoutc); p != nil; p = q.Pop(timeoutc) { + popped += len(p) + } + + testutil.Equals(t, qcapacity, popped) +} + func TestQueue_Push_Relabelled(t *testing.T) { q := NewQueue( nil, nil, 10, 10, From f87b9a188b9afcc7131d4ad2be7a3af1386a6b9c Mon Sep 17 00:00:00 2001 From: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com> Date: Sun, 8 Mar 2020 22:42:46 +0000 Subject: [PATCH 2/5] Fix alert queue bug by resignal after Pop (#2204) Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com> --- pkg/alert/alert.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/alert/alert.go b/pkg/alert/alert.go index 4ad30b7c03..d527b87abe 100644 --- a/pkg/alert/alert.go +++ b/pkg/alert/alert.go @@ -195,6 +195,12 @@ func (q *Queue) Pop(termc <-chan struct{}) []*Alert { q.popped.Add(float64(n)) + if len(q.queue) > 0 { + select { + case q.morec <- struct{}{}: + default: + } + } return as[:n] } From 07285c41ec7109a728c52fb258120e5383f2fec6 Mon Sep 17 00:00:00 2001 From: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com> Date: Sun, 8 Mar 2020 23:15:39 +0000 Subject: [PATCH 3/5] Fix alert queue test and simplify Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com> --- pkg/alert/alert_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/alert/alert_test.go b/pkg/alert/alert_test.go index d75265866b..67f280280d 100644 --- a/pkg/alert/alert_test.go +++ b/pkg/alert/alert_test.go @@ -20,7 +20,7 @@ import ( ) func TestQueue_Pop_all_Pushed(t *testing.T) { - qcapacity := 5 + qcapacity := 10 batchsize := 1 pushes := 3 @@ -34,13 +34,14 @@ func TestQueue_Pop_all_Pushed(t *testing.T) { }) } - timeoutc := time.After(time.Second) + timeoutc := make(chan struct{}, 1) + time.AfterFunc(time.Second, func() { timeoutc <- struct{}{} }) popped := 0 for p := q.Pop(timeoutc); p != nil; p = q.Pop(timeoutc) { popped += len(p) } - testutil.Equals(t, qcapacity, popped) + testutil.Equals(t, pushes*2, popped) } func TestQueue_Push_Relabelled(t *testing.T) { From a74567959a18c75f7478ac5ea16f0eef34ddce8f Mon Sep 17 00:00:00 2001 From: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com> Date: Mon, 9 Mar 2020 11:19:00 +0000 Subject: [PATCH 4/5] Update CHANGELOG.md Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c33357a391..fe59e1ba92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased +### Fixed + +- [#TODO](https://github.com/thanos-io/thanos/pull/#TODO) Ruler: Fixed Issue #2204 bug in alert queue signalling filled up queue and alerts were dropped + ## [v0.11.0](https://github.com/thanos-io/thanos/releases/tag/v0.11.0-rc.1) - 2020.03.02 ### Fixed From 3b0056c72a1dd0f6b3e27b01bb70d4a0b75b9b2e Mon Sep 17 00:00:00 2001 From: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com> Date: Mon, 9 Mar 2020 13:55:54 +0000 Subject: [PATCH 5/5] Link to thanos-io/thanos PR in CHANGELOG.md Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe59e1ba92..87b84f14d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Fixed -- [#TODO](https://github.com/thanos-io/thanos/pull/#TODO) Ruler: Fixed Issue #2204 bug in alert queue signalling filled up queue and alerts were dropped +- [#2238](https://github.com/thanos-io/thanos/pull/2238) Ruler: Fixed Issue #2204 bug in alert queue signalling filled up queue and alerts were dropped ## [v0.11.0](https://github.com/thanos-io/thanos/releases/tag/v0.11.0-rc.1) - 2020.03.02