lightning: every HTTP retry should use its own request (#47959)

close #47930

br/pkg/pdutil/pd.go

@@ -164,13 +164,16 @@ func pdRequestWithCode(
  return 0, nil, errors.Trace(err)
  }
  reqURL := fmt.Sprintf("%s/%s", u, prefix)
- req, err := http.NewRequestWithContext(ctx, method, reqURL, body)
- if err != nil {
- return 0, nil, errors.Trace(err)
- }
- var resp *http.Response
+ var (
+ req *http.Request
+ resp *http.Response
+ )
  count := 0
  for {
+ req, err = http.NewRequestWithContext(ctx, method, reqURL, body)
+ if err != nil {
+ return 0, nil, errors.Trace(err)
+ }
  resp, err = cli.Do(req) //nolint:bodyclose
  count++
  failpoint.Inject("InjectClosed", func(v failpoint.Value) {

br/pkg/pdutil/pd_serial_test.go

@@ -3,6 +3,7 @@
 package pdutil
 
 import (
+ "bytes"
  "context"
  "encoding/hex"
  "encoding/json"
@@ -186,8 +187,16 @@ func TestPDRequestRetry(t *testing.T) {
  w.WriteHeader(http.StatusOK)
  }))
  cli := http.DefaultClient
+ cli.Transport = http.DefaultTransport.(*http.Transport).Clone()
+ // although the real code doesn't disable keep alive, we need to disable it
+ // in test to avoid the connection being reused and #47930 can't appear. The
+ // real code will only meet #47930 when go's internal http client just dropped
+ // all idle connections.
+ cli.Transport.(*http.Transport).DisableKeepAlives = true
+
  taddr := ts.URL
- _, reqErr := pdRequest(ctx, taddr, "", cli, http.MethodGet, nil)
+ body := bytes.NewBuffer([]byte("test"))
+ _, reqErr := pdRequest(ctx, taddr, "", cli, http.MethodPost, body)
  require.NoError(t, reqErr)
  ts.Close()
  count = 0