Merge pull request PelicanPlatform#1116 from bbockelm/airplane_mode_v2

"Airplane mode" for unit tests

broker/broker_test.go

@@ -129,7 +129,9 @@ func doRetrieveRequest(t *testing.T, ctx context.Context, dur time.Duration) (*h
 
 	reqReader := bytes.NewReader(reqBytes)
 
-	brokerAud, err := url.Parse(param.Federation_BrokerUrl.GetString())
+	fedInfo, err := config.GetFederation(ctx)
+	require.NoError(t, err)
+	brokerAud, err := url.Parse(fedInfo.BrokerEndpoint)
 	require.NoError(t, err)
 	brokerAud.Path = ""
 

broker/client.go

@@ -549,7 +549,12 @@ func doCallback(ctx context.Context, brokerResp reversalRequest) (listener net.L
 // closes itself.  It is the result of a successful connection reversal to
 // a cache.
 func LaunchRequestMonitor(ctx context.Context, egrp *errgroup.Group, resultChan chan any) (err error) {
-	brokerUrl := param.Federation_BrokerUrl.GetString()
+	fedInfo, err := config.GetFederation(ctx)
+	if err != nil {
+		return err
+	}
+
+	brokerUrl := fedInfo.BrokerEndpoint
 	if brokerUrl == "" {
 		return errors.New("Broker service is not set or discovered; cannot enable broker functionality.  Try setting Federation.BrokerUrl")
 	}
@@ -588,7 +593,7 @@ func LaunchRequestMonitor(ctx context.Context, egrp *errgroup.Group, resultChan
 				req.Header.Set("Content-Type", "application/json")
 				req.Header.Set("User-Agent", "pelican-origin/"+config.GetVersion())
 
-				brokerAud, err := url.Parse(param.Federation_BrokerUrl.GetString())
+				brokerAud, err := url.Parse(fedInfo.BrokerEndpoint)
 				if err != nil {
 					log.Errorln("Failure when parsing broker URL:", err)
 					break

broker/token_utils.go

@@ -29,7 +29,6 @@ import (
 	"github.com/lestrrat-go/jwx/v2/jwk"
 	"github.com/lestrrat-go/jwx/v2/jwt"
 	"github.com/pelicanplatform/pelican/config"
-	"github.com/pelicanplatform/pelican/param"
 	"github.com/pelicanplatform/pelican/token"
 	"github.com/pelicanplatform/pelican/token_scopes"
 	"github.com/pkg/errors"
@@ -81,8 +80,11 @@ func LaunchNamespaceKeyMaintenance(ctx context.Context, egrp *errgroup.Group) {
 // Given a namespace prefix, return the value that should be used
 // by the `iss` claim in a token for this federation's registry.
 func getRegistryIssValue(prefix string) (iss string, err error) {
-	// Calculate the correct `iss` field as part of the registry service
-	namespaceUrlStr := param.Federation_RegistryUrl.GetString()
+	fedInfo, err := config.GetFederation(context.Background())
+	if err != nil {
+		return
+	}
+	namespaceUrlStr := fedInfo.NamespaceRegistrationEndpoint
 	if namespaceUrlStr == "" {
 		err = errors.New("namespace URL is not set")
 		return

broker/token_utils_test.go

@@ -23,13 +23,16 @@ import (
 	"time"
 
 	"github.com/lestrrat-go/jwx/v2/jwt"
+	"github.com/pelicanplatform/pelican/config"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestGetCacheHostnameFromToken(t *testing.T) {
 	viper.Reset()
+	config.InitConfig()
+	require.NoError(t, config.InitClient())
 
 	viper.Set("Federation.RegistryUrl", "https://your-registry.com")
 

cache/advertise.go

@@ -123,12 +123,15 @@ func (server *CacheServer) GetNamespaceAdsFromDirector() error {
 	// Get the endpoint of the director
 	var respNS []server_structs.NamespaceAdV2
 
-	directorEndpoint := param.Federation_DirectorUrl.GetString()
-	if directorEndpoint == "" {
+	fedInfo, err := config.GetFederation(context.Background())
+	if err != nil {
+		return err
+	}
+	if fedInfo.DirectorEndpoint == "" {
 		return errors.New("No director specified; give the federation name (-f)")
 	}
 
-	directorEndpointURL, err := url.Parse(directorEndpoint)
+	directorEndpointURL, err := url.Parse(fedInfo.DirectorEndpoint)
 	if err != nil {
 		return errors.Wrap(err, "Unable to parse director url")
 	}
@@ -145,7 +148,7 @@ func (server *CacheServer) GetNamespaceAdsFromDirector() error {
 	respData, err := utils.MakeRequest(context.Background(), directorNSListEndpointURL, "GET", nil, nil)
 	if err != nil {
 		if strings.Contains(err.Error(), "404") {
-			directorNSListEndpointURL, err = url.JoinPath(directorEndpoint, "api", "v1.0", "director", "listNamespaces")
+			directorNSListEndpointURL, err = url.JoinPath(fedInfo.DirectorEndpoint, "api", "v1.0", "director", "listNamespaces")
 			if err != nil {
 				return err
 			}

cache/advertise_test.go

@@ -24,6 +24,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/pelicanplatform/pelican/config"
 	"github.com/pelicanplatform/pelican/server_structs"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
@@ -122,15 +123,16 @@ func TestFilterNsAdsForCache(t *testing.T) {
 
 	for _, testInput := range tests {
 		t.Run(testInput.desc, func(t *testing.T) {
-
+			err := config.InitClient()
+			require.NoError(t, err)
 			viper.Set("Federation.DirectorURL", ts.URL)
 			if testInput.permittedNS != nil {
 				viper.Set("Cache.PermittedNamespaces", testInput.permittedNS)
 			}
 			defer viper.Reset()
 
 			cacheServer.SetFilters()
-			err := cacheServer.GetNamespaceAdsFromDirector()
+			err = cacheServer.GetNamespaceAdsFromDirector()
 			require.NoError(t, err)
 			filteredNS := cacheServer.GetNamespaceAds()
 

client/director.go

@@ -19,6 +19,7 @@
 package client
 
 import (
+	"context"
 	"encoding/json"
 	"io"
 	"net/http"
@@ -111,7 +112,7 @@ func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Name
 
 // Make a request to the director for a given verb/resource; return the
 // HTTP response object only if a 307 is returned.
-func queryDirector(verb, source, directorUrl string) (resp *http.Response, err error) {
+func queryDirector(ctx context.Context, verb, source, directorUrl string) (resp *http.Response, err error) {
 	resourceUrl := directorUrl + source
 	// Here we use http.Transport to prevent the client from following the director's
 	// redirect. We use the Location url elsewhere (plus we still need to do the token
@@ -125,7 +126,7 @@ func queryDirector(verb, source, directorUrl string) (resp *http.Response, err e
 		},
 	}
 
-	req, err := http.NewRequest(verb, resourceUrl, nil)
+	req, err := http.NewRequestWithContext(ctx, verb, resourceUrl, nil)
 	if err != nil {
 		log.Errorln("Failed to create an HTTP request:", err)
 		return nil, err

client/director_test.go

@@ -20,6 +20,7 @@ package client
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -198,7 +199,7 @@ func TestQueryDirector(t *testing.T) {
 	defer server.Close()
 
 	// Call QueryDirector with the test server URL and a source path
-	actualResp, err := queryDirector("GET", "/foo/bar", server.URL)
+	actualResp, err := queryDirector(context.Background(), "GET", "/foo/bar", server.URL)
 	if err != nil {
 		t.Fatal(err)
 	}

client/fed_linux_test.go

@@ -110,8 +110,12 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
 	innerTempFile.Close()
 
 	t.Run("testPelicanRecursiveGetAndPutPelicanURL", func(t *testing.T) {
-		_, err := config.SetPreferredPrefix(config.PelicanPrefix)
+		oldPref, err := config.SetPreferredPrefix(config.PelicanPrefix)
 		assert.NoError(t, err)
+		defer func() {
+			_, err := config.SetPreferredPrefix(oldPref)
+			require.NoError(t, err)
+		}()
 
 		for _, export := range fed.Exports {
 			// Set path for object to upload/download
@@ -196,7 +200,11 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
 	})
 
 	t.Run("testOsdfRecursiveGetAndPutOsdfURL", func(t *testing.T) {
-		_, err := config.SetPreferredPrefix(config.OsdfPrefix)
+		oldPref, err := config.SetPreferredPrefix(config.OsdfPrefix)
+		defer func() {
+			_, err := config.SetPreferredPrefix(oldPref)
+			require.NoError(t, err)
+		}()
 		assert.NoError(t, err)
 		for _, export := range fed.Exports {
 			// Set path for object to upload/download
@@ -289,8 +297,12 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
 	})
 
 	t.Run("testOsdfRecursiveGetAndPutPelicanURL", func(t *testing.T) {
-		_, err := config.SetPreferredPrefix(config.OsdfPrefix)
+		oldPref, err := config.SetPreferredPrefix(config.OsdfPrefix)
 		assert.NoError(t, err)
+		defer func() {
+			_, err := config.SetPreferredPrefix(oldPref)
+			require.NoError(t, err)
+		}()
 
 		for _, export := range fed.Exports {
 			// Set path for object to upload/download

client/fed_test.go

@@ -103,7 +103,11 @@ func TestGetAndPutAuth(t *testing.T) {
 
 	// This tests object get/put with a pelican:// url
 	t.Run("testPelicanObjectPutAndGetWithPelicanUrl", func(t *testing.T) {
-		_, err := config.SetPreferredPrefix(config.PelicanPrefix)
+		oldPref, err := config.SetPreferredPrefix(config.PelicanPrefix)
+		defer func() {
+			_, err := config.SetPreferredPrefix(oldPref)
+			require.NoError(t, err)
+		}()
 		assert.NoError(t, err)
 
 		// Set path for object to upload/download
@@ -131,8 +135,12 @@ func TestGetAndPutAuth(t *testing.T) {
 
 	// This tests object get/put with a pelican:// url
 	t.Run("testOsdfObjectPutAndGetWithPelicanUrl", func(t *testing.T) {
-		_, err := config.SetPreferredPrefix(config.OsdfPrefix)
+		oldPref, err := config.SetPreferredPrefix(config.OsdfPrefix)
 		assert.NoError(t, err)
+		defer func() {
+			_, err := config.SetPreferredPrefix(oldPref)
+			require.NoError(t, err)
+		}()
 
 		for _, export := range fed.Exports {
 			// Set path for object to upload/download
@@ -159,8 +167,12 @@ func TestGetAndPutAuth(t *testing.T) {
 
 	// This tests pelican object get/put with an osdf url
 	t.Run("testOsdfObjectPutAndGetWithOSDFUrl", func(t *testing.T) {
-		_, err := config.SetPreferredPrefix(config.OsdfPrefix)
+		oldPref, err := config.SetPreferredPrefix(config.OsdfPrefix)
 		assert.NoError(t, err)
+		defer func() {
+			_, err := config.SetPreferredPrefix(oldPref)
+			require.NoError(t, err)
+		}()
 
 		for _, export := range fed.Exports {
 			// Set path for object to upload/download
@@ -247,8 +259,12 @@ func TestCopyAuth(t *testing.T) {
 
 	// This tests object get/put with a pelican:// url
 	t.Run("testPelicanObjectCopyWithPelicanUrl", func(t *testing.T) {
-		_, err := config.SetPreferredPrefix(config.PelicanPrefix)
+		oldPref, err := config.SetPreferredPrefix(config.PelicanPrefix)
 		assert.NoError(t, err)
+		defer func() {
+			_, err := config.SetPreferredPrefix(oldPref)
+			require.NoError(t, err)
+		}()
 
 		// Set path for object to upload/download
 		for _, export := range fed.Exports {
@@ -275,8 +291,12 @@ func TestCopyAuth(t *testing.T) {
 
 	// This tests object get/put with a pelican:// url
 	t.Run("testOsdfObjectCopyWithPelicanUrl", func(t *testing.T) {
-		_, err := config.SetPreferredPrefix(config.OsdfPrefix)
+		oldPref, err := config.SetPreferredPrefix(config.OsdfPrefix)
 		assert.NoError(t, err)
+		defer func() {
+			_, err := config.SetPreferredPrefix(oldPref)
+			require.NoError(t, err)
+		}()
 
 		for _, export := range fed.Exports {
 			// Set path for object to upload/download
@@ -303,8 +323,12 @@ func TestCopyAuth(t *testing.T) {
 
 	// This tests pelican object get/put with an osdf url
 	t.Run("testOsdfObjectCopyWithOSDFUrl", func(t *testing.T) {
-		_, err := config.SetPreferredPrefix(config.OsdfPrefix)
+		oldPref, err := config.SetPreferredPrefix(config.OsdfPrefix)
 		assert.NoError(t, err)
+		defer func() {
+			_, err := config.SetPreferredPrefix(oldPref)
+			require.NoError(t, err)
+		}()
 
 		for _, export := range fed.Exports {
 			// Set path for object to upload/download
@@ -420,8 +444,12 @@ func TestStatHttp(t *testing.T) {
 	})
 
 	t.Run("testStatHttpOSDFScheme", func(t *testing.T) {
-		_, err := config.SetPreferredPrefix(config.OsdfPrefix)
+		oldPref, err := config.SetPreferredPrefix(config.OsdfPrefix)
 		assert.NoError(t, err)
+		defer func() {
+			_, err := config.SetPreferredPrefix(oldPref)
+			require.NoError(t, err)
+		}()
 		testFileContent := "test file content"
 		// Drop the testFileContent into the origin directory
 		tempFile, err := os.Create(filepath.Join(fed.Exports[0].StoragePrefix, "test.txt"))

client/handle_http.go

@@ -508,10 +508,14 @@ func (te *TransferEngine) newPelicanURL(remoteUrl *url.URL) (pelicanURL pelicanU
 		// If we are using an osdf/stash binary, we discovered the federation already --> load into local url metadata
 		if config.GetPreferredPrefix() == config.OsdfPrefix {
 			log.Debugln("In OSDF mode with osdf:// url; populating metadata with OSDF defaults")
-			if param.Federation_DirectorUrl.GetString() == "" || param.Federation_DiscoveryUrl.GetString() == "" || param.Federation_RegistryUrl.GetString() == "" {
+			fedInfo, err := config.GetFederation(te.ctx)
+			if fedInfo.DirectorEndpoint == "" {
+				if err != nil {
+					return pelicanUrl{}, errors.Wrap(err, "no OSDF metadata available")
+				}
 				return pelicanUrl{}, fmt.Errorf("OSDF default metadata is not populated in config")
 			} else {
-				pelicanURL.directorUrl = param.Federation_DirectorUrl.GetString()
+				pelicanURL.directorUrl = fedInfo.DirectorEndpoint
 			}
 		} else if config.GetPreferredPrefix() == config.PelicanPrefix {
 			// We hit this case when we are using a pelican binary but an osdf:// url, therefore we need to disover the osdf federation
@@ -528,21 +532,24 @@ func (te *TransferEngine) newPelicanURL(remoteUrl *url.URL) (pelicanURL pelicanU
 	} else if scheme == "pelican" && remoteUrl.Host == "" {
 		// We hit this case when we do not have a hostname with a pelican:// url
 		if param.Federation_DiscoveryUrl.GetString() == "" {
-			return pelicanUrl{}, fmt.Errorf("Pelican url scheme without discovery-url detected, please provide a federation discovery-url " +
+			return pelicanUrl{}, errors.Errorf("pelican url scheme without discovery-url detected, please provide a federation discovery-url " +
 				"(e.g. pelican://<federation-url-in-hostname.org></namespace></path/to/file>) within the hostname or with the -f flag")
+		}
+		// Check if cache has key of federationURL, if not, loader will add it:
+		pelicanUrlItem := te.pelicanURLCache.Get(param.Federation_DiscoveryUrl.GetString())
+		if pelicanUrlItem != nil {
+			pelicanURL = pelicanUrlItem.Value().url
 		} else {
-			// Check if cache has key of federationURL, if not, loader will add it:
-			pelicanUrlItem := te.pelicanURLCache.Get(param.Federation_DiscoveryUrl.GetString())
-			if pelicanUrlItem != nil {
-				pelicanURL = pelicanUrlItem.Value().url
-			} else {
-				return pelicanUrl{}, fmt.Errorf("Issue getting metadata information from cache")
-			}
+			return pelicanUrl{}, errors.Errorf("issue getting metadata information from cache")
 		}
 	} else if scheme == "" {
 		// If we don't have a url scheme, then our metadata information should be in the config
 		log.Debugln("No url scheme detected, getting metadata information from configuration")
-		pelicanURL.directorUrl = param.Federation_DirectorUrl.GetString()
+		if fedInfo, err := config.GetFederation(te.ctx); err == nil {
+			pelicanURL.directorUrl = fedInfo.DirectorEndpoint
+		} else {
+			return pelicanUrl{}, errors.Wrap(err, "failed to lookup pelican metadata from configuration")
+		}
 
 		// If the values do not exist, exit with failure
 		if pelicanURL.directorUrl == "" {
@@ -1040,7 +1047,7 @@ func (tc *TransferClient) NewTransferJob(ctx context.Context, remoteUrl *url.URL
 	}
 
 	tj.useDirector = pelicanURL.directorUrl != ""
-	ns, err := getNamespaceInfo(remoteUrl.Path, pelicanURL.directorUrl, upload)
+	ns, err := getNamespaceInfo(tj.ctx, remoteUrl.Path, pelicanURL.directorUrl, upload)
 	if err != nil {
 		log.Errorln(err)
 		err = errors.Wrapf(err, "failed to get namespace information for remote URL %s", remoteUrl.String())