diff --git a/client/director.go b/client/director.go index 9df8545f5..bf8377e14 100644 --- a/client/director.go +++ b/client/director.go @@ -74,6 +74,7 @@ func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Name namespace.UseTokenOnRead, _ = strconv.ParseBool(xPelicanNamespace["require-token"]) namespace.ReadHTTPS, _ = strconv.ParseBool(xPelicanNamespace["readhttps"]) namespace.DirListHost = xPelicanNamespace["collections-url"] + namespace.PutEndpoint = xPelicanNamespace["putendpoint"] var xPelicanAuthorization map[string]string if len(dirResp.Header.Values("X-Pelican-Authorization")) > 0 { @@ -125,7 +126,11 @@ func CreateNsFromDirectorResp(dirResp *http.Response) (namespace namespaces.Name } func QueryDirector(source string, directorUrl string) (resp *http.Response, err error) { - resourceUrl := directorUrl + source + resourceUrl, err := url.JoinPath(directorUrl, source) + if err != nil { + return nil, err + } + // Here we use http.Transport to prevent the client from following the director's // redirect. We use the Location url elsewhere (plus we still need to do the token // dance!) diff --git a/client/handle_http.go b/client/handle_http.go index ebcdfb63c..ea4af3f71 100644 --- a/client/handle_http.go +++ b/client/handle_http.go @@ -835,13 +835,13 @@ func UploadFile(src string, origDest *url.URL, token string, namespace namespace } // Parse the writeback host as a URL - writebackhostUrl, err := url.Parse(namespace.WriteBackHost) + putEndpointUrl, err := url.Parse(namespace.PutEndpoint) if err != nil { return 0, err } dest := &url.URL{ - Host: writebackhostUrl.Host, + Host: putEndpointUrl.Host, Scheme: "https", Path: origDest.Path, } @@ -1089,11 +1089,11 @@ func StatHttp(dest *url.URL, namespace namespaces.Namespace) (uint64, error) { } // Parse the writeback host as a URL - writebackhostUrl, err := url.Parse(namespace.WriteBackHost) + putEndpointUrl, err := url.Parse(namespace.PutEndpoint) if err != nil { return 0, err } - dest.Host = writebackhostUrl.Host + dest.Host = putEndpointUrl.Host dest.Scheme = "https" canDisableProxy := CanDisableProxy() diff --git a/client/handle_http_test.go b/client/handle_http_test.go index ebc6b7686..84061466a 100644 --- a/client/handle_http_test.go +++ b/client/handle_http_test.go @@ -389,7 +389,7 @@ func TestFullUpload(t *testing.T) { testURL, err := url.Parse(ts.URL) assert.NoError(t, err, "Error parsing test URL") testNamespace := namespaces.Namespace{ - WriteBackHost: "https://" + testURL.Host, + PutEndpoint: "https://" + testURL.Host, } // Upload the file diff --git a/client/main.go b/client/main.go index 63529c566..b935b10bf 100644 --- a/client/main.go +++ b/client/main.go @@ -498,13 +498,34 @@ func DoStashCPSingle(sourceFile string, destination string, methods []string, re // For write back, it will be the destination // For read it will be the source. + OSDFDirectorUrl := param.Federation_DirectorUrl.GetString() + useOSDFDirector := viper.IsSet("Federation.DirectorURL") + if destScheme == "stash" || destScheme == "osdf" || destScheme == "pelican" { log.Debugln("Detected writeback") - ns, err := namespaces.MatchNamespace(dest_url.Path) - if err != nil { - log.Errorln("Failed to get namespace information:", err) - AddError(err) - return 0, err + if !strings.HasPrefix(destination, "/") { + destination = strings.TrimPrefix(destination, destScheme+"://") + } + var ns namespaces.Namespace + // If we have a director set, go through that for namespace info, otherwise use topology + if useOSDFDirector { + dirResp, err := QueryDirector(destination, OSDFDirectorUrl) + if err != nil { + log.Errorln("Error while querying the Director:", err) + AddError(err) + return 0, err + } + ns, err = CreateNsFromDirectorResp(dirResp) + if err != nil { + AddError(err) + return 0, err + } + } else { + ns, err = namespaces.MatchNamespace(dest_url.Path) + if err != nil { + AddError(err) + return 0, err + } } _, err = doWriteBack(source_url.Path, dest_url, ns, recursive) AddError(err) @@ -523,10 +544,8 @@ func DoStashCPSingle(sourceFile string, destination string, methods []string, re sourceFile = "/" + sourceFile } - OSDFDirectorUrl := param.Federation_DirectorUrl.GetString() - useOSDFDirector := viper.IsSet("Federation.DirectorURL") - var ns namespaces.Namespace + // If we have a director set, go through that for namespace info, otherwise use topology if useOSDFDirector { dirResp, err := QueryDirector(sourceFile, OSDFDirectorUrl) if err != nil { diff --git a/cmd/director_serve.go b/cmd/director_serve.go index e29084f9c..f7bdeb528 100644 --- a/cmd/director_serve.go +++ b/cmd/director_serve.go @@ -60,8 +60,8 @@ func serveDirector( /*cmd*/ *cobra.Command /*args*/, []string) error { if err := director.AdvertiseOSDF(); err != nil { panic(err) } + go director.PeriodicCacheReload() } - go director.PeriodicCacheReload() director.ConfigTTLCache(shutdownCtx, &wg) wg.Add(1) // Add to wait group after ConfigTTLCache finishes to avoid deadlock diff --git a/director/advertise.go b/director/advertise.go index b8d520301..29055c797 100644 --- a/director/advertise.go +++ b/director/advertise.go @@ -78,6 +78,7 @@ func AdvertiseOSDF() error { nsAd.RequireToken = ns.UseTokenOnRead nsAd.Path = ns.Path nsAd.DirlistHost = ns.DirlistHost + nsAd.PutEndpoint = ns.PutEndpoint issuerURL, err := url.Parse(ns.CredentialGeneration.Issuer) if err != nil { log.Warningf("Invalid URL %v when parsing topology response: %v\n", ns.CredentialGeneration.Issuer, err) diff --git a/director/cache_ads.go b/director/cache_ads.go index d54a867c8..f74a50066 100644 --- a/director/cache_ads.go +++ b/director/cache_ads.go @@ -43,6 +43,7 @@ type ( BasePath string `json:"basePath"` VaultServer string `json:"vaultServer"` DirlistHost string `json:"dirlisthost"` + PutEndpoint string `json:"writebackhost"` } ServerAd struct { diff --git a/director/redirect.go b/director/redirect.go index 8de5faf50..120847668 100644 --- a/director/redirect.go +++ b/director/redirect.go @@ -227,8 +227,8 @@ func RedirectToCache(ginCtx *gin.Context) { ginCtx.Writer.Header()["X-Pelican-Token-Generation"] = []string{tokenGen} } } - ginCtx.Writer.Header()["X-Pelican-Namespace"] = []string{fmt.Sprintf("namespace=%s, require-token=%v, collections-url=%s", - namespaceAd.Path, namespaceAd.RequireToken, namespaceAd.DirlistHost)} + ginCtx.Writer.Header()["X-Pelican-Namespace"] = []string{fmt.Sprintf("namespace=%s, require-token=%v, collections-url=%s, putendpoint=%s", + namespaceAd.Path, namespaceAd.RequireToken, namespaceAd.DirlistHost, namespaceAd.PutEndpoint)} // Note we only append the `authz` query parameter in the case of the redirect response and not the // duplicate link metadata above. This is purposeful: the Link header might get too long if we repeat diff --git a/namespaces/namespaces.go b/namespaces/namespaces.go index 6fe91b80b..120eb18f7 100644 --- a/namespaces/namespaces.go +++ b/namespaces/namespaces.go @@ -86,7 +86,7 @@ type Namespace struct { Issuer string `json:"issuer"` ReadHTTPS bool `json:"readhttps"` UseTokenOnRead bool `json:"usetokenonread"` - WriteBackHost string `json:"writebackhost"` + PutEndpoint string `json:"writebackhost"` DirListHost string `json:"dirlisthost"` } diff --git a/namespaces/namespaces_test.go b/namespaces/namespaces_test.go index edebb2a10..52de1059b 100644 --- a/namespaces/namespaces_test.go +++ b/namespaces/namespaces_test.go @@ -175,7 +175,7 @@ func TestFullNamespace(t *testing.T) { assert.Equal(t, true, ns.ReadHTTPS) assert.Equal(t, true, ns.UseTokenOnRead) assert.Equal(t, "/ospool/PROTECTED", ns.Path) - assert.Equal(t, "https://origin-auth2001.chtc.wisc.edu:1095", ns.WriteBackHost) + assert.Equal(t, "https://origin-auth2001.chtc.wisc.edu:1095", ns.PutEndpoint) } diff --git a/origin_ui/advertise.go b/origin_ui/advertise.go index d05b1eeea..4805a9a5b 100644 --- a/origin_ui/advertise.go +++ b/origin_ui/advertise.go @@ -62,6 +62,8 @@ func (server *OriginServer) CreateAdvertisement(name string, originUrl string, o MaxScopeDepth: 3, Strategy: "OAuth2", BasePath: prefix, + // TODO: This is set to the origin for now needs to become configurable + PutEndpoint: originUrl, } ad = director.OriginAdvertise{ Name: name, diff --git a/utils/web_utils.go b/utils/web_utils.go index 843a66b26..97b98fba9 100644 --- a/utils/web_utils.go +++ b/utils/web_utils.go @@ -56,7 +56,7 @@ type ( Path string `json:"path"` ReadHTTPS bool `json:"readhttps"` UseTokenOnRead bool `json:"usetokenonread"` - WritebackHost string `json:"writebackhost"` + PutEndpoint string `json:"writebackhost"` } TopologyNamespacesJSON struct {