feat: refactor Fetcher interface used for downloading migrations

repo/fsrepo/migrations/migrations.go

@@ -155,13 +155,14 @@ func ReadMigrationConfig(repoRoot string) (*config.Migration, error) {
 // downloadSources,
 func GetMigrationFetcher(downloadSources []string, distPath string, newIpfsFetcher func(string) Fetcher) (Fetcher, error) {
  const httpUserAgent = "go-ipfs"
+ const numRetriesPerHTTP = 3
 
  var fetchers []Fetcher
  for _, src := range downloadSources {
  src := strings.TrimSpace(src)
  switch src {
  case "HTTPS", "https", "HTTP", "http":
- fetchers = append(fetchers, NewHttpFetcher(distPath, "", httpUserAgent, 0))
+ fetchers = append(fetchers, NewRetryFetcher(NewHttpFetcher(distPath, "", httpUserAgent, 0), numRetriesPerHTTP))
  case "IPFS", "ipfs":
  if newIpfsFetcher != nil {
  fetchers = append(fetchers, newIpfsFetcher(distPath))
@@ -178,7 +179,7 @@ func GetMigrationFetcher(downloadSources []string, distPath string, newIpfsFetch
  default:
  return nil, errors.New("bad gateway address: url scheme must be http or https")
  }
- fetchers = append(fetchers, NewHttpFetcher(distPath, u.String(), httpUserAgent, 0))
+ fetchers = append(fetchers, NewRetryFetcher(NewHttpFetcher(distPath, u.String(), httpUserAgent, 0), numRetriesPerHTTP))
  case "":
  // Ignore empty string
  }

repo/fsrepo/migrations/migrations_test.go

@@ -325,7 +325,9 @@ func TestGetMigrationFetcher(t *testing.T) {
  if err != nil {
  t.Fatal(err)
  }
- if _, ok := f.(*HttpFetcher); !ok {
+ if rf, ok := f.(*RetryFetcher); !ok {
+ t.Fatal("expected RetryFetcher")
+ } else if _, ok := rf.Fetcher.(*HttpFetcher); !ok {
  t.Fatal("expected HttpFetcher")
  }
 
@@ -343,7 +345,9 @@ func TestGetMigrationFetcher(t *testing.T) {
  if err != nil {
  t.Fatal(err)
  }
- if _, ok := f.(*HttpFetcher); !ok {
+ if rf, ok := f.(*RetryFetcher); !ok {
+ t.Fatal("expected RetryFetcher")
+ } else if _, ok := rf.Fetcher.(*HttpFetcher); !ok {
  t.Fatal("expected HttpFetcher")
  }
 

repo/fsrepo/migrations/retryfetcher.go

@@ -0,0 +1,43 @@
+package migrations
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+)
+
+type RetryFetcher struct {
+ Fetcher
+ maxRetries int
+}
+
+var _ Fetcher = (*RetryFetcher)(nil)
+
+func NewRetryFetcher(baseFetcher Fetcher, maxRetries int) *RetryFetcher {
+ return &RetryFetcher{Fetcher: baseFetcher, maxRetries: maxRetries}
+}
+
+func (r *RetryFetcher) Fetch(ctx context.Context, filePath string, writer io.Writer) error {
+ var lastErr error
+ for i := 0; i < r.maxRetries; i++ {
+ var buf bytes.Buffer
+ err := r.Fetcher.Fetch(ctx, filePath, &buf)
+ if err == nil {
+ if _, err := io.Copy(writer, &buf); err != nil {
+ return err
+ }
+ return nil
+ }
+
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ lastErr = err
+ }
+ return fmt.Errorf("exceeded number of retries. last error was %w", lastErr)
+}
+
+func (r *RetryFetcher) Close() error {
+ return r.Fetcher.Close()
+}