Merge pull request #183 from joereuss12/transport-fix-and-generated-a…

…fter-configs-branch

Pelican transport fixes and generated after configs

client/director.go

@@ -25,6 +25,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/pelicanplatform/pelican/config"
 	namespaces "github.com/pelicanplatform/pelican/namespaces"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -121,7 +122,7 @@ func QueryDirector(source string, directorUrl string) (resp *http.Response, err
 	// redirect. We use the Location url elsewhere (plus we still need to do the token
 	// dance!)
 	var client *http.Client
-	tr := GetTransport()
+	tr := config.GetTransport()
 	client = &http.Client{
 		Transport: tr,
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {

client/handle_http.go

@@ -20,7 +20,6 @@ package client
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
@@ -45,17 +44,12 @@ import (
 	"github.com/vbauerster/mpb/v8"
 	"github.com/vbauerster/mpb/v8/decor"
 
+	"github.com/pelicanplatform/pelican/config"
 	namespaces "github.com/pelicanplatform/pelican/namespaces"
-	"github.com/pelicanplatform/pelican/param"
 )
 
 var p = mpb.New()
 
-var (
-	transport     *http.Transport
-	onceTransport sync.Once
-)
-
 type StoppedTransferError struct {
 	Err string
 }
@@ -445,49 +439,12 @@ func parseTransferStatus(status string) (int, string) {
 	return statusCode, strings.TrimSpace(parts[1])
 }
 
-func setupTransport() *http.Transport {
-	//Getting timeouts and other information from defaults.yaml
-	maxIdleConns := viper.GetInt("Transport.MaxIdleIcons")
-	idleConnTimeout := viper.GetDuration("Transport.IdleConnTimeout")
-	transportTLSHandshakeTimeout := viper.GetDuration("Transport.TLSHandshakeTimeout")
-	expectContinueTimeout := viper.GetDuration("Transport.ExpectContinueTimeout")
-	responseHeaderTimeout := viper.GetDuration("Transport.ResponseHeaderTimeout")
-
-	transportDialerTimeout := viper.GetDuration("Transport.Dialer.Timeout")
-	transportKeepAlive := viper.GetDuration("Transport.Dialer.KeepAlive")
-
-	//Set up the transport
-	return &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   transportDialerTimeout,
-			KeepAlive: transportKeepAlive,
-		}).DialContext,
-		MaxIdleConns:          maxIdleConns,
-		IdleConnTimeout:       idleConnTimeout,
-		TLSHandshakeTimeout:   transportTLSHandshakeTimeout,
-		ExpectContinueTimeout: expectContinueTimeout,
-		ResponseHeaderTimeout: responseHeaderTimeout,
-	}
-}
-
-// function to get/setup the transport (only once)
-func GetTransport() *http.Transport {
-	onceTransport.Do(func() {
-		transport = setupTransport()
-	})
-	if param.TLSSkipVerify.GetBool() {
-		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-	}
-	return transport
-}
-
 // DownloadHTTP - Perform the actual download of the file
 func DownloadHTTP(transfer TransferDetails, dest string, token string) (int64, error) {
 
 	// Create the client, request, and context
 	client := grab.NewClient()
-	transport := GetTransport()
+	transport := config.GetTransport()
 	if !transfer.Proxy {
 		transport.Proxy = nil
 	}
@@ -799,7 +756,7 @@ Loop:
 
 }
 
-var UploadClient = &http.Client{Transport: GetTransport()}
+var UploadClient = &http.Client{Transport: config.GetTransport()}
 
 // Actually perform the Put request to the server
 func doPut(request *http.Request, responseChan chan<- *http.Response, errorChan chan<- error) {
@@ -850,7 +807,7 @@ func walkDavDir(url *url.URL, namespace namespaces.Namespace) ([]string, error)
 	c := gowebdav.NewClient(rootUrl.String(), "", "")
 
 	// XRootD does not like keep alives and kills things, so turn them off.
-	transport = GetTransport()
+	transport := config.GetTransport()
 	c.SetTransport(transport)
 
 	files, err := walkDir(url.Path, c)
@@ -902,7 +859,7 @@ func StatHttp(dest *url.URL, namespace namespaces.Namespace) (uint64, error) {
 
 	var resp *http.Response
 	for {
-		transport := GetTransport()
+		transport := config.GetTransport()
 		if disableProxy {
 			log.Debugln("Performing HEAD (without proxy)", dest.String())
 			transport.Proxy = nil

config/config.go

@@ -19,10 +19,12 @@
 package config
 
 import (
+	"crypto/tls"
 	_ "embed"
 	"encoding/json"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -86,6 +88,10 @@ var (
 	// Potentially holds a directory to cleanup
 	tempRunDir  string
 	cleanupOnce sync.Once
+
+	// Our global transports that only will get reconfigured if needed
+	transport     *http.Transport
+	onceTransport sync.Once
 )
 
 // Based on the name of the current binary, determine the preferred "style"
@@ -236,6 +242,44 @@ func getConfigBase() (string, error) {
 	return filepath.Join(home, ".config", "pelican"), nil
 }
 
+func setupTransport() {
+	//Getting timeouts and other information from defaults.yaml
+	maxIdleConns := viper.GetInt("Transport.MaxIdleIcons")
+	idleConnTimeout := viper.GetDuration("Transport.IdleConnTimeout")
+	transportTLSHandshakeTimeout := viper.GetDuration("Transport.TLSHandshakeTimeout")
+	expectContinueTimeout := viper.GetDuration("Transport.ExpectContinueTimeout")
+	responseHeaderTimeout := viper.GetDuration("Transport.ResponseHeaderTimeout")
+
+	transportDialerTimeout := viper.GetDuration("Transport.Dialer.Timeout")
+	transportKeepAlive := viper.GetDuration("Transport.Dialer.KeepAlive")
+
+	//Set up the transport
+	transport = &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   transportDialerTimeout,
+			KeepAlive: transportKeepAlive,
+		}).DialContext,
+		MaxIdleConns:          maxIdleConns,
+		IdleConnTimeout:       idleConnTimeout,
+		TLSHandshakeTimeout:   transportTLSHandshakeTimeout,
+		ExpectContinueTimeout: expectContinueTimeout,
+		ResponseHeaderTimeout: responseHeaderTimeout,
+	}
+	if param.TLSSkipVerify.GetBool() {
+		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+
+}
+
+// function to get/setup the transport (only once)
+func GetTransport() *http.Transport {
+	onceTransport.Do(func() {
+		setupTransport()
+	})
+	return transport
+}
+
 func InitServer() error {
 	configDir := viper.GetString("ConfigDir")
 	viper.SetConfigType("yaml")
@@ -314,6 +358,9 @@ func InitServer() error {
 			return err
 		}
 	}
+
+	setupTransport()
+
 	return nil
 }
 
@@ -419,5 +466,7 @@ func InitClient() error {
 	}
 	viper.Set("MinimumDownloadSpeed", downloadLimit)
 
+	setupTransport()
+
 	return DiscoverFederation()
 }

namespace-registry/client_commands.go

@@ -34,7 +34,6 @@ import (
 	"github.com/lestrrat-go/jwx/v2/jwa"
 	"github.com/lestrrat-go/jwx/v2/jwk"
 	"github.com/lestrrat-go/jwx/v2/jwt"
-	"github.com/pelicanplatform/pelican/client"
 	"github.com/pelicanplatform/pelican/config"
 	"github.com/pelicanplatform/pelican/director"
 	log "github.com/sirupsen/logrus"
@@ -62,8 +61,7 @@ func makeRequest(url string, method string, data map[string]interface{}, headers
 	for key, val := range headers {
 		req.Header.Set(key, val)
 	}
-
-	tr := client.GetTransport()
+	tr := config.GetTransport()
 	client := &http.Client{Transport: tr}
 
 	resp, err := client.Do(req)

origin_ui/advertise.go

@@ -26,7 +26,7 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/pelicanplatform/pelican/client"
+	"github.com/pelicanplatform/pelican/config"
 	"github.com/pelicanplatform/pelican/director"
 	"github.com/pelicanplatform/pelican/param"
 	"github.com/pkg/errors"
@@ -122,7 +122,7 @@ func AdvertiseOrigin() error {
 
 	// We should switch this over to use the common transport, but for that to happen
 	// that function needs to be exported from pelican
-	tr := client.GetTransport()
+	tr := config.GetTransport()
 	client := http.Client{Transport: tr}
 
 	resp, err := client.Do(req)