Skip to content

Commit

Permalink
fix(otel): Map 500 errors to 503 (#13173)
Browse files Browse the repository at this point in the history
(cherry picked from commit b31e04e)
  • Loading branch information
salvacorts authored and grafana-delivery-bot[bot] committed Jun 7, 2024
1 parent 6e66e68 commit 6d5ce34
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 1 deletion.
21 changes: 20 additions & 1 deletion pkg/distributor/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,26 @@ func (d *Distributor) PushHandler(w http.ResponseWriter, r *http.Request) {
}

func (d *Distributor) OTLPPushHandler(w http.ResponseWriter, r *http.Request) {
d.pushHandler(w, r, push.ParseOTLPRequest)
interceptor := newOtelErrorHeaderInterceptor(w)
d.pushHandler(interceptor, r, push.ParseOTLPRequest)
}

// otelErrorHeaderInterceptor maps 500 errors to 503.
// According to the OTLP specification, 500 errors are never retried on the client side, but 503 are.
type otelErrorHeaderInterceptor struct {
http.ResponseWriter
}

func newOtelErrorHeaderInterceptor(w http.ResponseWriter) *otelErrorHeaderInterceptor {
return &otelErrorHeaderInterceptor{ResponseWriter: w}
}

func (i *otelErrorHeaderInterceptor) WriteHeader(statusCode int) {
if statusCode == http.StatusInternalServerError {
statusCode = http.StatusServiceUnavailable
}

i.ResponseWriter.WriteHeader(statusCode)
}

func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRequestParser push.RequestParser) {
Expand Down
32 changes: 32 additions & 0 deletions pkg/distributor/http_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,38 @@ func TestRequestParserWrapping(t *testing.T) {
require.True(t, called)
}

func Test_OtelErrorHeaderInterceptor(t *testing.T) {
for _, tc := range []struct {
name string
inputCode int
expectedCode int
}{
{
name: "500",
inputCode: http.StatusInternalServerError,
expectedCode: http.StatusServiceUnavailable,
},
{
name: "400",
inputCode: http.StatusBadRequest,
expectedCode: http.StatusBadRequest,
},
{
name: "204",
inputCode: http.StatusNoContent,
expectedCode: http.StatusNoContent,
},
} {
t.Run(tc.name, func(t *testing.T) {
r := httptest.NewRecorder()
i := newOtelErrorHeaderInterceptor(r)

http.Error(i, "error", tc.inputCode)
require.Equal(t, tc.expectedCode, r.Code)
})
}
}

func stubParser(_ string, _ *http.Request, _ push.TenantsRetention, _ push.Limits, _ push.UsageTracker) (*logproto.PushRequest, *push.Stats, error) {
return &logproto.PushRequest{}, &push.Stats{}, nil
}

0 comments on commit 6d5ce34

Please sign in to comment.