fix: add retry and try to fix network errors such as EOF

Author: 0x2E

pkg/httpx/httpx.go

@@ -38,7 +38,7 @@ func NewSafeClient() *http.Client {
 	safeTransport := &http.Transport{
 		DialContext:       safeDialer.DialContext,
 		ForceAttemptHTTP2: true,
-		// DisableKeepAlives: true,
+		DisableKeepAlives: true,
 	}
 	return &http.Client{
 		Transport: safeTransport,

service/pull/pull.go

@@ -3,6 +3,7 @@ package pull
 import (
 	"context"
 	"errors"
+	"net"
 	"sync"
 	"time"
 
@@ -79,8 +80,17 @@ func (p *Puller) PullAll(ctx context.Context, force bool) error {
 				<-routinePool
 			}()
 
+			logger := logger.With("feed_id", f.ID)
 			if err := p.do(ctx, f, force); err != nil {
-				logger.Errorw(err.Error(), "feed_id", f.ID)
+				logger.Errorln(err)
+				if _, ok := err.(net.Error); ok {
+					for i := 1; i < 4; i++ {
+						logger.Infof("%dth retry", i)
+						if p.do(ctx, f, true) == nil {
+							break
+						}
+					}
+				}
 			}
 		}(f)
 	}

service/sniff/request.go

@@ -19,6 +19,8 @@ func request(ctx context.Context, link string) (*http.Response, error) {
 		return nil, err
 	}
 
+	req.Close = true
+
 	ua := "fusion/1.0"
 	req.Header.Add("User-Agent", ua)