From 875e2adc40a0488518b12944baec6c082b78d806 Mon Sep 17 00:00:00 2001 From: "Nicholas (Nick) Meyer" Date: Mon, 4 Oct 2021 09:59:53 -0700 Subject: [PATCH] Improve github pull request call retries (#1810) * Improve github pull request call retries Retry with fixed 1 second backoff up to 3 retries was added by #1131 to address #1019, but the issue continued to show up (#1453). Increase max attempts to 5 and use exponential backoff for a maximum total retry time of (2^n - n - 1) seconds, which is roughly 30 seconds for current max attempts n = 5. Also move the sleep to the top of the loop so that we never sleep without sending the request again on the last iteration. * Fix style with gofmt -s --- server/events/vcs/github_client.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/server/events/vcs/github_client.go b/server/events/vcs/github_client.go index f55a9e7882..117a2d3fcb 100644 --- a/server/events/vcs/github_client.go +++ b/server/events/vcs/github_client.go @@ -292,10 +292,14 @@ func (g *GithubClient) GetPullRequest(repo models.Repo, num int) (*github.PullRe // GitHub has started to return 404's here (#1019) even after they send the webhook. // They've got some eventual consistency issues going on so we're just going - // to retry up to 3 times with a 1s sleep. - numRetries := 3 - retryDelay := 1 * time.Second - for i := 0; i < numRetries; i++ { + // to attempt up to 5 times with exponential backoff. + maxAttempts := 5 + attemptDelay := 0 * time.Second + for i := 0; i < maxAttempts; i++ { + // First don't sleep, then sleep 1, 3, 7, etc. + time.Sleep(attemptDelay) + attemptDelay = 2*attemptDelay + 1*time.Second + pull, _, err = g.client.PullRequests.Get(g.ctx, repo.Owner, repo.Name, num) if err == nil { return pull, nil @@ -304,7 +308,6 @@ func (g *GithubClient) GetPullRequest(repo models.Repo, num int) (*github.PullRe if !ok || ghErr.Response.StatusCode != 404 { return pull, err } - time.Sleep(retryDelay) } return pull, err }