Skip to content

Commit

Permalink
Retry more errors
Browse files Browse the repository at this point in the history
  • Loading branch information
prymitive committed Oct 26, 2021
1 parent 1290c3d commit 6dc82d4
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 1 deletion.
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# Changelog

## [v0.1.4]

### Changed

- Retry queries that error with `query processing would load too many samples into memory`
using a smaller time range.

## [v0.1.3]

### Added
Expand Down
13 changes: 12 additions & 1 deletion internal/promapi/range.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net"
"strings"
"time"

"github.com/prometheus/client_golang/api"
Expand Down Expand Up @@ -64,7 +65,7 @@ func RangeQuery(uri string, timeout time.Duration, expr string, start, end time.
Msg("Range query completed")
if err != nil {
log.Error().Err(err).Str("uri", uri).Str("query", expr).Msg("Range query failed")
if err, ok := err.(net.Error); ok && err.Timeout() {
if isRetryable(err) {
delta := end.Sub(start) / 2
log.Warn().Str("delta", HumanizeDuration(delta)).Msg("Retrying request with smaller range")
b, _ := start.MarshalText()
Expand Down Expand Up @@ -95,3 +96,13 @@ func RangeQuery(uri string, timeout time.Duration, expr string, start, end time.

return &qr, nil
}

func isRetryable(err error) bool {
if err, ok := err.(net.Error); ok && err.Timeout() {
return true
}
if strings.Contains(err.Error(), "query processing would load too many samples into memory in ") {
return true
}
return false
}

0 comments on commit 6dc82d4

Please sign in to comment.