Skip to content

Commit

Permalink
fix flaky query frontend split interval unit test (#3110)
Browse files Browse the repository at this point in the history 
Signed-off-by: Ben Ye <yb532204897@gmail.com>
  • Loading branch information
@yeya24
yeya24 authored Sep 2, 2020
1 parent 2502d02 commit 4bf2447
Showing 1 changed file with 59 additions and 88 deletions.
147 changes: 59 additions & 88 deletions pkg/queryfrontend/roundtrip_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,11 @@ func TestRoundTripRetryMiddleware(t *testing.T) {
codec := NewThanosCodec(true)

for _, tc := range []struct {
name string
maxRetries int
req queryrange.Request
handlerAndResult func() (*int, http.Handler)
expectedError bool
expected int
name string
maxRetries int
req queryrange.Request
fail bool
expected int
}{
{
name: "not query range, retry won't be triggered 1.",
Expand All @@ -102,10 +101,7 @@ func TestRoundTripRetryMiddleware(t *testing.T) {
End: 2 * hour,
Step: 10 * seconds,
},
handlerAndResult: counter,
// Not go through tripperware so no error.
expectedError: false,
expected: 1,
expected: 1,
},
{
name: "not query range, retry won't be triggered 2.",
Expand All @@ -116,34 +112,28 @@ func TestRoundTripRetryMiddleware(t *testing.T) {
End: 2 * hour,
Step: 10 * seconds,
},
handlerAndResult: counter,
// Not go through tripperware so no error.
expectedError: false,
expected: 1,
expected: 1,
},
{
name: "no retry, get counter value 1",
maxRetries: 0,
req: testRequest,
handlerAndResult: counter,
expectedError: true,
expected: 1,
name: "no retry, get counter value 1",
maxRetries: 0,
req: testRequest,
fail: true,
expected: 1,
},
{
name: "retry set to 1",
maxRetries: 1,
req: testRequest,
handlerAndResult: counter,
expectedError: true,
expected: 1,
name: "retry set to 1",
maxRetries: 1,
req: testRequest,
fail: true,
expected: 1,
},
{
name: "retry set to 3",
maxRetries: 3,
req: testRequest,
handlerAndResult: counter,
expectedError: true,
expected: 3,
name: "retry set to 3",
maxRetries: 3,
fail: true,
req: testRequest,
expected: 3,
},
} {

Expand All @@ -154,15 +144,15 @@ func TestRoundTripRetryMiddleware(t *testing.T) {

rt, err := newFakeRoundTripper()
testutil.Ok(t, err)
res, handler := tc.handlerAndResult()
res, handler := promqlResults(tc.fail)
rt.setHandler(handler)

ctx := user.InjectOrgID(context.Background(), "1")
httpReq, err := codec.EncodeRequest(ctx, tc.req)
testutil.Ok(t, err)

_, err = tpw(rt).RoundTrip(httpReq)
testutil.Equals(t, tc.expectedError, err != nil)
testutil.Equals(t, tc.fail, err != nil)

testutil.Equals(t, tc.expected, *res)
})
Expand All @@ -182,12 +172,10 @@ func TestRoundTripSplitIntervalMiddleware(t *testing.T) {
codec := NewThanosCodec(true)

for _, tc := range []struct {
name string
splitInterval time.Duration
req queryrange.Request
handlerAndResult func() (*int, http.Handler)
expectError bool
expected int
name string
splitInterval time.Duration
req queryrange.Request
expected int
}{
{
name: "non query range request won't be split 1",
Expand All @@ -197,10 +185,8 @@ func TestRoundTripSplitIntervalMiddleware(t *testing.T) {
End: 2 * hour,
Step: 10 * seconds,
},
splitInterval: time.Hour,
handlerAndResult: counter,
expectError: false,
expected: 1,
splitInterval: time.Hour,
expected: 1,
},
{
name: "non query range request won't be split 2",
Expand All @@ -210,42 +196,32 @@ func TestRoundTripSplitIntervalMiddleware(t *testing.T) {
End: 2 * hour,
Step: 10 * seconds,
},
splitInterval: time.Hour,
handlerAndResult: counter,
expectError: false,
expected: 1,
splitInterval: time.Hour,
expected: 1,
},
{
name: "split interval == 0, disable split",
req: testRequest,
splitInterval: 0,
handlerAndResult: counter,
expectError: true,
expected: 1,
name: "split interval == 0, disable split",
req: testRequest,
splitInterval: 0,
expected: 1,
},
{
name: "won't be split. Interval == time range",
req: testRequest,
splitInterval: 2 * time.Hour,
handlerAndResult: counter,
expectError: true,
expected: 1,
name: "won't be split. Interval == time range",
req: testRequest,
splitInterval: 2 * time.Hour,
expected: 1,
},
{
name: "won't be split. Interval > time range",
req: testRequest,
splitInterval: day,
handlerAndResult: counter,
expectError: true,
expected: 1,
name: "won't be split. Interval > time range",
req: testRequest,
splitInterval: day,
expected: 1,
},
{
name: "split to 2 requests",
req: testRequest,
splitInterval: 1 * time.Hour,
handlerAndResult: counter,
expectError: true,
expected: 2,
name: "split to 2 requests",
req: testRequest,
splitInterval: 1 * time.Hour,
expected: 2,
},
} {

Expand All @@ -256,15 +232,16 @@ func TestRoundTripSplitIntervalMiddleware(t *testing.T) {

rt, err := newFakeRoundTripper()
testutil.Ok(t, err)
res, handler := tc.handlerAndResult()
defer rt.Close()
res, handler := promqlResults(false)
rt.setHandler(handler)

ctx := user.InjectOrgID(context.Background(), "1")
httpReq, err := codec.EncodeRequest(ctx, tc.req)
testutil.Ok(t, err)

_, err = tpw(rt).RoundTrip(httpReq)
testutil.Equals(t, tc.expectError, err != nil)
testutil.Ok(t, err)

testutil.Equals(t, tc.expected, *res)
})
Expand Down Expand Up @@ -330,7 +307,8 @@ func TestRoundTripCacheMiddleware(t *testing.T) {

rt, err := newFakeRoundTripper()
testutil.Ok(t, err)
res, handler := promqlResults()
defer rt.Close()
res, handler := promqlResults(false)
rt.setHandler(handler)

for _, tc := range []struct {
Expand Down Expand Up @@ -382,21 +360,9 @@ func TestRoundTripCacheMiddleware(t *testing.T) {
}
}

// counter is a mock handler used to test retry and split.
// Copied from Loki https://github.com/grafana/loki/blob/master/pkg/querier/queryrange/roundtrip_test.go#L526.
func counter() (*int, http.Handler) {
count := 0
var lock sync.Mutex
return &count, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
lock.Lock()
defer lock.Unlock()
count++
})
}

// promqlResults is a mock handler used to test cache middleware.
// Modified from Loki https://github.com/grafana/loki/blob/master/pkg/querier/queryrange/roundtrip_test.go#L547.
func promqlResults() (*int, http.Handler) {
func promqlResults(fail bool) (*int, http.Handler) {
count := 0
var lock sync.Mutex
q := queryrange.PrometheusResponse{
Expand All @@ -418,6 +384,11 @@ func promqlResults() (*int, http.Handler) {
return &count, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
lock.Lock()
defer lock.Unlock()

// Set fail in the response code to test retry.
if fail {
w.WriteHeader(500)
}
if err := json.NewEncoder(w).Encode(q); err != nil {
panic(err)
}
Expand Down

0 comments on commit 4bf2447

Please sign in to comment.