Pelican transport fixes and generated after configs

Fixed an issue with common transports for some reason not getting correct value from defaults.yaml. Also addressed issue moving the common transports to the config sub-module and allowed initClient and initServer to recreate the common transport object. In addition, moved the TLSSkipVerify to only run when setupTransfer is called again (either first time `GetTransfer()` is called or `initClient()` or `initServer()` is called).

client/director.go

@@ -25,6 +25,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/pelicanplatform/pelican/config"
 	namespaces "github.com/pelicanplatform/pelican/namespaces"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -121,7 +122,7 @@ func QueryDirector(source string, directorUrl string) (resp *http.Response, err
 	// redirect. We use the Location url elsewhere (plus we still need to do the token
 	// dance!)
 	var client *http.Client
-	tr := GetTransport()
+	tr := config.GetTransport()
 	client = &http.Client{
 		Transport: tr,
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {

client/handle_http.go

@@ -20,7 +20,6 @@ package client
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -45,17 +44,12 @@ import (
 	"github.com/vbauerster/mpb/v8"
 	"github.com/vbauerster/mpb/v8/decor"
 
+	"github.com/pelicanplatform/pelican/config"
 	namespaces "github.com/pelicanplatform/pelican/namespaces"
-	"github.com/pelicanplatform/pelican/param"
 )
 
 var p = mpb.New()
 
-var (
-	transport     *http.Transport
-	onceTransport sync.Once
-)
-
 type StoppedTransferError struct {
 	Err string
 }
@@ -445,49 +439,12 @@ func parseTransferStatus(status string) (int, string) {
 	return statusCode, strings.TrimSpace(parts[1])
 }
 
-func setupTransport() *http.Transport {
-	//Getting timeouts and other information from defaults.yaml
-	maxIdleConns := viper.GetInt("Transport.MaxIdleIcons")
-	idleConnTimeout := viper.GetDuration("Transport.IdleConnTimeout")
-	transportTLSHandshakeTimeout := viper.GetDuration("Transport.TLSHandshakeTimeout")
-	expectContinueTimeout := viper.GetDuration("Transport.ExpectContinueTimeout")
-	responseHeaderTimeout := viper.GetDuration("Transport.ResponseHeaderTimeout")
-
-	transportDialerTimeout := viper.GetDuration("Transport.Dialer.Timeout")
-	transportKeepAlive := viper.GetDuration("Transport.Dialer.KeepAlive")
-
-	//Set up the transport
-	return &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   transportDialerTimeout,
-			KeepAlive: transportKeepAlive,
-		}).DialContext,
-		MaxIdleConns:          maxIdleConns,
-		IdleConnTimeout:       idleConnTimeout,
-		TLSHandshakeTimeout:   transportTLSHandshakeTimeout,
-		ExpectContinueTimeout: expectContinueTimeout,
-		ResponseHeaderTimeout: responseHeaderTimeout,
-	}
-}
-
-// function to get/setup the transport (only once)
-func GetTransport() *http.Transport {
-	onceTransport.Do(func() {
-		transport = setupTransport()
-	})
-	if param.TLSSkipVerify.GetBool() {
-		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-	}
-	return transport
-}
-
 // DownloadHTTP - Perform the actual download of the file
 func DownloadHTTP(transfer TransferDetails, dest string, token string) (int64, error) {
 
 	// Create the client, request, and context
 	client := grab.NewClient()
-	transport := GetTransport()
+	transport := config.GetTransport()
 	if !transfer.Proxy {
 		transport.Proxy = nil
 	}
@@ -799,7 +756,7 @@ Loop:
 
 }
 
-var UploadClient = &http.Client{Transport: GetTransport()}
+var UploadClient = &http.Client{Transport: config.GetTransport()}
 
 // Actually perform the Put request to the server
 func doPut(request *http.Request, responseChan chan<- *http.Response, errorChan chan<- error) {
@@ -850,7 +807,7 @@ func walkDavDir(url *url.URL, namespace namespaces.Namespace) ([]string, error)
 	c := gowebdav.NewClient(rootUrl.String(), "", "")
 
 	// XRootD does not like keep alives and kills things, so turn them off.
-	transport = GetTransport()
+	transport := config.GetTransport()
 	c.SetTransport(transport)
 
 	files, err := walkDir(url.Path, c)
@@ -902,7 +859,7 @@ func StatHttp(dest *url.URL, namespace namespaces.Namespace) (uint64, error) {
 
 	var resp *http.Response
 	for {
-		transport := GetTransport()
+		transport := config.GetTransport()
 		if disableProxy {
 			log.Debugln("Performing HEAD (without proxy)", dest.String())
 			transport.Proxy = nil

config/config.go

@@ -19,10 +19,12 @@
 package config
 
 import (
+	"crypto/tls"
 	_ "embed"
 	"encoding/json"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -86,6 +88,10 @@ var (
 	// Potentially holds a directory to cleanup
 	tempRunDir  string
 	cleanupOnce sync.Once
+
+	// Our global transports that only will get reconfigured if needed
+	transport     *http.Transport
+	onceTransport sync.Once
 )
 
 // Based on the name of the current binary, determine the preferred "style"
@@ -236,6 +242,45 @@ func getConfigBase() (string, error) {
 	return filepath.Join(home, ".config", "pelican"), nil
 }
 
+func setupTransport() {
+	//Getting timeouts and other information from defaults.yaml
+	maxIdleConns := viper.GetInt("Transport.MaxIdleIcons")
+	idleConnTimeout := viper.GetDuration("Transport.IdleConnTimeout")
+	transportTLSHandshakeTimeout := viper.GetDuration("Transport.TLSHandshakeTimeout")
+	expectContinueTimeout := viper.GetDuration("Transport.ExpectContinueTimeout")
+	responseHeaderTimeout := viper.GetDuration("Transport.ResponseHeaderTimeout")
+
+	transportDialerTimeout := viper.GetDuration("Transport.Dialer.Timeout")
+	transportKeepAlive := viper.GetDuration("Transport.Dialer.KeepAlive")
+
+	//Set up the transport
+	transport = &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   transportDialerTimeout,
+			KeepAlive: transportKeepAlive,
+		}).DialContext,
+		MaxIdleConns:          maxIdleConns,
+		IdleConnTimeout:       idleConnTimeout,
+		TLSHandshakeTimeout:   transportTLSHandshakeTimeout,
+		ExpectContinueTimeout: expectContinueTimeout,
+		ResponseHeaderTimeout: responseHeaderTimeout,
+	}
+	if param.TLSSkipVerify.GetBool() {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+	return
+}
+
+// function to get/setup the transport (only once)
+func GetTransport() *http.Transport {
+	onceTransport.Do(func() {
+		setupTransport()
+	})
+	return transport
+}
+
 func InitServer() error {
 	configDir := viper.GetString("ConfigDir")
 	viper.SetConfigType("yaml")
@@ -314,6 +359,9 @@ func InitServer() error {
 			return err
 		}
 	}
+
+	setupTransport()
+
 	return nil
 }
 
@@ -419,5 +467,7 @@ func InitClient() error {
 	}
 	viper.Set("MinimumDownloadSpeed", downloadLimit)
 
+	setupTransport()
+
 	return DiscoverFederation()
 }

namespace-registry/client_commands.go

@@ -34,7 +34,6 @@ import (
 	"github.com/lestrrat-go/jwx/v2/jwa"
 	"github.com/lestrrat-go/jwx/v2/jwk"
 	"github.com/lestrrat-go/jwx/v2/jwt"
-	"github.com/pelicanplatform/pelican/client"
 	"github.com/pelicanplatform/pelican/config"
 	"github.com/pelicanplatform/pelican/director"
 	log "github.com/sirupsen/logrus"
@@ -62,8 +61,7 @@ func makeRequest(url string, method string, data map[string]interface{}, headers
 	for key, val := range headers {
 		req.Header.Set(key, val)
 	}
-
-	tr := client.GetTransport()
+	tr := config.GetTransport()
 	client := &http.Client{Transport: tr}
 
 	resp, err := client.Do(req)

origin_ui/advertise.go

@@ -26,7 +26,7 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/pelicanplatform/pelican/client"
+	"github.com/pelicanplatform/pelican/config"
 	"github.com/pelicanplatform/pelican/director"
 	"github.com/pelicanplatform/pelican/param"
 	"github.com/pkg/errors"
@@ -122,7 +122,7 @@ func AdvertiseOrigin() error {
 
 	// We should switch this over to use the common transport, but for that to happen
 	// that function needs to be exported from pelican
-	tr := client.GetTransport()
+	tr := config.GetTransport()
 	client := http.Client{Transport: tr}
 
 	resp, err := client.Do(req)