-
Notifications
You must be signed in to change notification settings - Fork 17
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add retries to all resources and datasources #286
Changes from all commits
10e71e8
7449a34
87d25bc
1946bf8
e4b78ac
ced91a8
bd16224
280141f
61e9a09
3b32f83
f86254b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,7 @@ | ||
package client | ||
|
||
import ( | ||
"context" | ||
"errors" | ||
"fmt" | ||
"net/http" | ||
|
@@ -10,14 +11,38 @@ import ( | |
"github.com/google/uuid" | ||
|
||
"github.com/prefecthq/terraform-provider-prefect/internal/api" | ||
|
||
retryablehttp "github.com/hashicorp/go-retryablehttp" | ||
) | ||
|
||
var _ = api.PrefectClient(&Client{}) | ||
|
||
// New creates and returns new client instance. | ||
func New(opts ...Option) (*Client, error) { | ||
// Uses the retryablehttp package for built-in retries | ||
// with exponential backoff. | ||
// | ||
// Some notable defaults from that package include: | ||
// - max retries: 4 | ||
// - retry wait minimum seconds: 1 | ||
// - retry wait maximum seconds: 30 | ||
// | ||
// All defaults are defined in | ||
// https://github.com/hashicorp/go-retryablehttp/blob/main/client.go#L48-L51. | ||
retryableClient := retryablehttp.NewClient() | ||
|
||
// By default, retryablehttp will only retry requests if there was some kind | ||
// of transient server or networking error. We can be more specific with this | ||
// by providing a custom function for determining whether or not to retry. | ||
retryableClient.CheckRetry = checkRetryPolicy | ||
|
||
// Finally, convert the retryablehttp client to a standard http client. | ||
// This allows us to retain the `http.Client` interface, and avoid specifying | ||
// the `retryablehttp.Client` interface in our client methods. | ||
httpClient := retryableClient.StandardClient() | ||
|
||
client := &Client{ | ||
hc: http.DefaultClient, | ||
hc: httpClient, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is the main change here - we're replacing the default HTTP client with our retry-able HTTP client. |
||
} | ||
|
||
var errs []error | ||
|
@@ -36,26 +61,6 @@ func New(opts ...Option) (*Client, error) { | |
return client, nil | ||
} | ||
|
||
// MustNew returns a new client or panics if an error occurred. | ||
func MustNew(opts ...Option) *Client { | ||
client, err := New(opts...) | ||
if err != nil { | ||
panic(fmt.Sprintf("error occurred during construction: %s", err)) | ||
} | ||
|
||
return client | ||
} | ||
|
||
// WithClient configures the underlying http.Client used to send | ||
// requests. | ||
func WithClient(httpClient *http.Client) Option { | ||
return func(client *Client) error { | ||
client.hc = httpClient | ||
|
||
return nil | ||
} | ||
} | ||
|
||
Comment on lines
-39
to
-58
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These were unused. |
||
// WithEndpoint configures the client to communicate with a self-hosted | ||
// Prefect server or Prefect Cloud. | ||
func WithEndpoint(endpoint string) Option { | ||
|
@@ -97,3 +102,21 @@ func WithDefaults(accountID uuid.UUID, workspaceID uuid.UUID) Option { | |
return nil | ||
} | ||
} | ||
|
||
func checkRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { | ||
// If the response is a 409 (StatusConflict), that means the request | ||
// eventually succeeded and we don't need to make the request again. | ||
if resp.StatusCode == http.StatusConflict { | ||
return false, nil | ||
} | ||
|
||
// If the response is a 404 (NotFound), try again. This is particularly | ||
// relevant for block-related objects that are created asynchronously. | ||
if resp.StatusCode == http.StatusNotFound { | ||
return true, err | ||
} | ||
|
||
// Fall back to the default retry policy for any other status codes. | ||
//nolint:wrapcheck // we've extended this method, no need to wrap error | ||
return retryablehttp.DefaultRetryPolicy(ctx, resp, err) | ||
} | ||
Comment on lines
+105
to
+122
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Here we can be specific about what warrants a retry. This is cool because we knew in the past we didn't need to retry 409s, but the retry implementation we had at the time didn't really let us achieve that. |
This file was deleted.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This part is pretty sweet. It saves a lot of lines changed - see f86254b.