Origins now advertise themselves as writebackhosts

client/director.go

@@ -74,6 +74,7 @@ func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Name
 	namespace.UseTokenOnRead, _ = strconv.ParseBool(xPelicanNamespace["require-token"])
 	namespace.ReadHTTPS, _ = strconv.ParseBool(xPelicanNamespace["readhttps"])
 	namespace.DirListHost = xPelicanNamespace["collections-url"]
+	namespace.PutEndpoint = xPelicanNamespace["putendpoint"]
 
 	var xPelicanAuthorization map[string]string
 	if len(dirResp.Header.Values("X-Pelican-Authorization")) > 0 {
@@ -125,7 +126,11 @@ func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Name
 }
 
 func QueryDirector(source string, directorUrl string) (resp *http.Response, err error) {
-	resourceUrl := directorUrl + source
+	resourceUrl, err := url.JoinPath(directorUrl, source)
+	if err != nil {
+		return nil, err
+	}
+
 	// Here we use http.Transport to prevent the client from following the director's
 	// redirect. We use the Location url elsewhere (plus we still need to do the token
 	// dance!)

client/handle_http.go

@@ -835,13 +835,13 @@ func UploadFile(src string, origDest *url.URL, token string, namespace namespace
 	}
 
 	// Parse the writeback host as a URL
-	writebackhostUrl, err := url.Parse(namespace.WriteBackHost)
+	putEndpointUrl, err := url.Parse(namespace.PutEndpoint)
 	if err != nil {
 		return 0, err
 	}
 
 	dest := &url.URL{
-		Host:   writebackhostUrl.Host,
+		Host:   putEndpointUrl.Host,
 		Scheme: "https",
 		Path:   origDest.Path,
 	}
@@ -1089,11 +1089,11 @@ func StatHttp(dest *url.URL, namespace namespaces.Namespace) (uint64, error) {
 	}
 
 	// Parse the writeback host as a URL
-	writebackhostUrl, err := url.Parse(namespace.WriteBackHost)
+	putEndpointUrl, err := url.Parse(namespace.PutEndpoint)
 	if err != nil {
 		return 0, err
 	}
-	dest.Host = writebackhostUrl.Host
+	dest.Host = putEndpointUrl.Host
 	dest.Scheme = "https"
 
 	canDisableProxy := CanDisableProxy()

client/handle_http_test.go

@@ -389,7 +389,7 @@ func TestFullUpload(t *testing.T) {
 	testURL, err := url.Parse(ts.URL)
 	assert.NoError(t, err, "Error parsing test URL")
 	testNamespace := namespaces.Namespace{
-		WriteBackHost: "https://" + testURL.Host,
+		PutEndpoint: "https://" + testURL.Host,
 	}
 
 	// Upload the file

client/main.go

@@ -498,13 +498,34 @@ func DoStashCPSingle(sourceFile string, destination string, methods []string, re
 	// For write back, it will be the destination
 	// For read it will be the source.
 
+	OSDFDirectorUrl := param.Federation_DirectorUrl.GetString()
+	useOSDFDirector := viper.IsSet("Federation.DirectorURL")
+
 	if destScheme == "stash" || destScheme == "osdf" || destScheme == "pelican" {
 		log.Debugln("Detected writeback")
-		ns, err := namespaces.MatchNamespace(dest_url.Path)
-		if err != nil {
-			log.Errorln("Failed to get namespace information:", err)
-			AddError(err)
-			return 0, err
+		if !strings.HasPrefix(destination, "/") {
+			destination = strings.TrimPrefix(destination, destScheme+"://")
+		}
+		var ns namespaces.Namespace
+		// If we have a director set, go through that for namespace info, otherwise use topology
+		if useOSDFDirector {
+			dirResp, err := QueryDirector(destination, OSDFDirectorUrl)
+			if err != nil {
+				log.Errorln("Error while querying the Director:", err)
+				AddError(err)
+				return 0, err
+			}
+			ns, err = CreateNsFromDirectorResp(dirResp)
+			if err != nil {
+				AddError(err)
+				return 0, err
+			}
+		} else {
+			ns, err = namespaces.MatchNamespace(dest_url.Path)
+			if err != nil {
+				AddError(err)
+				return 0, err
+			}
 		}
 		_, err = doWriteBack(source_url.Path, dest_url, ns, recursive)
 		AddError(err)
@@ -523,10 +544,8 @@ func DoStashCPSingle(sourceFile string, destination string, methods []string, re
 		sourceFile = "/" + sourceFile
 	}
 
-	OSDFDirectorUrl := param.Federation_DirectorUrl.GetString()
-	useOSDFDirector := viper.IsSet("Federation.DirectorURL")
-
 	var ns namespaces.Namespace
+	// If we have a director set, go through that for namespace info, otherwise use topology
 	if useOSDFDirector {
 		dirResp, err := QueryDirector(sourceFile, OSDFDirectorUrl)
 		if err != nil {

cmd/director_serve.go

@@ -60,8 +60,8 @@ func serveDirector( /*cmd*/ *cobra.Command /*args*/, []string) error {
 		if err := director.AdvertiseOSDF(); err != nil {
 			panic(err)
 		}
+		go director.PeriodicCacheReload()
 	}
-	go director.PeriodicCacheReload()
 
 	director.ConfigTTLCache(shutdownCtx, &wg)
 	wg.Add(1) // Add to wait group after ConfigTTLCache finishes to avoid deadlock

director/advertise.go

@@ -78,6 +78,7 @@ func AdvertiseOSDF() error {
 		nsAd.RequireToken = ns.UseTokenOnRead
 		nsAd.Path = ns.Path
 		nsAd.DirlistHost = ns.DirlistHost
+		nsAd.PutEndpoint = ns.PutEndpoint
 		issuerURL, err := url.Parse(ns.CredentialGeneration.Issuer)
 		if err != nil {
 			log.Warningf("Invalid URL %v when parsing topology response: %v\n", ns.CredentialGeneration.Issuer, err)

director/cache_ads.go

@@ -43,6 +43,7 @@ type (
 		BasePath      string       `json:"basePath"`
 		VaultServer   string       `json:"vaultServer"`
 		DirlistHost   string       `json:"dirlisthost"`
+		PutEndpoint   string       `json:"writebackhost"`
 	}
 
 	ServerAd struct {

director/redirect.go

@@ -227,8 +227,8 @@ func RedirectToCache(ginCtx *gin.Context) {
 			ginCtx.Writer.Header()["X-Pelican-Token-Generation"] = []string{tokenGen}
 		}
 	}
-	ginCtx.Writer.Header()["X-Pelican-Namespace"] = []string{fmt.Sprintf("namespace=%s, require-token=%v, collections-url=%s",
-		namespaceAd.Path, namespaceAd.RequireToken, namespaceAd.DirlistHost)}
+	ginCtx.Writer.Header()["X-Pelican-Namespace"] = []string{fmt.Sprintf("namespace=%s, require-token=%v, collections-url=%s, putendpoint=%s",
+		namespaceAd.Path, namespaceAd.RequireToken, namespaceAd.DirlistHost, namespaceAd.PutEndpoint)}
 
 	// Note we only append the `authz` query parameter in the case of the redirect response and not the
 	// duplicate link metadata above.  This is purposeful: the Link header might get too long if we repeat

namespaces/namespaces.go

@@ -86,7 +86,7 @@ type Namespace struct {
 	Issuer               string                `json:"issuer"`
 	ReadHTTPS            bool                  `json:"readhttps"`
 	UseTokenOnRead       bool                  `json:"usetokenonread"`
-	WriteBackHost        string                `json:"writebackhost"`
+	PutEndpoint          string                `json:"writebackhost"`
 	DirListHost          string                `json:"dirlisthost"`
 }
 

namespaces/namespaces_test.go

@@ -175,7 +175,7 @@ func TestFullNamespace(t *testing.T) {
 	assert.Equal(t, true, ns.ReadHTTPS)
 	assert.Equal(t, true, ns.UseTokenOnRead)
 	assert.Equal(t, "/ospool/PROTECTED", ns.Path)
-	assert.Equal(t, "https://origin-auth2001.chtc.wisc.edu:1095", ns.WriteBackHost)
+	assert.Equal(t, "https://origin-auth2001.chtc.wisc.edu:1095", ns.PutEndpoint)
 
 }
 

origin_ui/advertise.go

@@ -62,6 +62,8 @@ func (server *OriginServer) CreateAdvertisement(name string, originUrl string, o
 		MaxScopeDepth: 3,
 		Strategy:      "OAuth2",
 		BasePath:      prefix,
+		// TODO: This is set to the origin for now needs to become configurable
+		PutEndpoint: originUrl,
 	}
 	ad = director.OriginAdvertise{
 		Name:       name,

utils/web_utils.go

@@ -56,7 +56,7 @@ type (
 		Path                 string               `json:"path"`
 		ReadHTTPS            bool                 `json:"readhttps"`
 		UseTokenOnRead       bool                 `json:"usetokenonread"`
-		WritebackHost        string               `json:"writebackhost"`
+		PutEndpoint          string               `json:"writebackhost"`
 	}
 
 	TopologyNamespacesJSON struct {