Skip to content

Commit

Permalink
Remove use of global fed metadata within client
Browse files Browse the repository at this point in the history 
This is v2 of PR PelicanPlatform#798 since the merge conflicts were becoming very
tedious. Therefore, I took the changes from that PR and converted it
over to a new branch. You can get the changes split up a little better
from that PR as well but I will do my best to list them here:

Changes include:
- Added a ttl cache for storing federation url metadata so we do not
  have to look up each time
- Improved unit tests for client commands, Note: for pelican prefix and
  osdf:// url scheme, it's hard to test since we use osg-htc.org for
  metadata lookup. Therefore it is hard to work with fed in a box so I
  just check for the proper metadata lookup
- Added function called discoverUrlFederation which discovers a
  federation from a url and does not populate global metadata config
  values
- Added a function called schemeUnderstood to take some repeated code
  within client into one function to check for understood url schemes
- Added unit tests for discoverUrlFederation and schemeUnderstood
  • Loading branch information
@joereuss12
joereuss12 committed Apr 1, 2024
1 parent da0e767 commit bb8f89d
Show file tree
Hide file tree
Showing 12 changed files with 650 additions and 491 deletions.
134 changes: 40 additions & 94 deletions client/fed_linux_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ package client_test
import (
"context"
"fmt"
"net/url"
"os"
"path"
"path/filepath"
Expand All @@ -45,7 +46,6 @@ import (

func TestRecursiveUploadsAndDownloads(t *testing.T) {
// Create instance of test federation
ctx, _, _ := test_utils.TestContext(context.Background(), t)
viper.Reset()
server_utils.ResetOriginExports()

Expand Down Expand Up @@ -100,97 +100,36 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
tempFile2.Close()

t.Run("testPelicanRecursiveGetAndPutOsdfURL", func(t *testing.T) {
config.SetPreferredPrefix("pelican")
for _, export := range *fed.Exports {
// Set path for object to upload/download
tempPath := tempDir
dirName := filepath.Base(tempPath)
// Note: minimally fixing this test as it is soon to be replaced
uploadURL := fmt.Sprintf("pelican://%s:%s%s/%s/%s", param.Server_Hostname.GetString(), strconv.Itoa(param.Server_WebPort.GetInt()),
export.FederationPrefix, "pel_osdf", dirName)

//////////////////////////////////////////////////////////
config.SetPreferredPrefix("PELICAN")
// Set path for object to upload/download
tempPath := tempDir
dirName := filepath.Base(tempPath)
uploadStr := "osdf:///test/" + dirName
uploadURL, err := url.Parse(uploadStr)
assert.NoError(t, err)

// Upload the file with PUT
transferDetailsUpload, err := client.DoPut(ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
require.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytes17 := 0
countBytes23 := 0
// Verify we got the correct files back (have to do this since files upload in different orders at times)
for _, transfer := range transferDetailsUpload {
transferredBytes := transfer.TransferredBytes
switch transferredBytes {
case int64(17):
countBytes17++
continue
case int64(23):
countBytes23++
continue
default:
// We got a byte amount we are not expecting
t.Fatal("did not upload proper amount of bytes")
}
}
if countBytes17 != 1 || countBytes23 != 1 {
// We would hit this case if 1 counter got hit twice for some reason
t.Fatal("One of the files was not uploaded correctly")
}
} else if len(transferDetailsUpload) != 2 {
t.Fatalf("Amount of transfers results returned for upload was not correct. Transfer details returned: %d", len(transferDetailsUpload))
}
// For OSDF url's, we don't want to rely on osdf metadata to be running therefore, just ensure we get correct metadata for the url:
pelicanURL, err := client.NewPelicanURL(uploadURL, "osdf")
assert.NoError(t, err)

// Download the files we just uploaded
var transferDetailsDownload []client.TransferResults
if export.Capabilities.PublicReads {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true)
} else {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true, client.WithTokenLocation(tempToken.Name()))
}
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytesUploadIdx0 := 0
countBytesUploadIdx1 := 0
// Verify we got the correct files back (have to do this since files upload in different orders at times)
// In this case, we want to match them to the sizes of the uploaded files
for _, transfer := range transferDetailsUpload {
transferredBytes := transfer.TransferredBytes
switch transferredBytes {
case transferDetailsUpload[0].TransferredBytes:
countBytesUploadIdx0++
continue
case transferDetailsUpload[1].TransferredBytes:
countBytesUploadIdx1++
continue
default:
// We got a byte amount we are not expecting
t.Fatal("did not download proper amount of bytes")
}
}
if countBytesUploadIdx0 != 1 || countBytesUploadIdx1 != 1 {
// We would hit this case if 1 counter got hit twice for some reason
t.Fatal("One of the files was not downloaded correctly")
} else if len(transferDetailsDownload) != 2 {
t.Fatalf("Amount of transfers results returned for download was not correct. Transfer details returned: %d", len(transferDetailsDownload))
}
}

}
// Check valid metadata:
assert.Equal(t, "https://osdf-director.osg-htc.org", pelicanURL.DirectorUrl)
assert.Equal(t, "https://osdf-registry.osg-htc.org", pelicanURL.RegistryUrl)
assert.Equal(t, "osg-htc.org", pelicanURL.DiscoveryUrl)
})

t.Run("testPelicanRecursiveGetAndPutPelicanURL", func(t *testing.T) {
config.SetPreferredPrefix("pelican")
config.SetPreferredPrefix("PELICAN")

for _, export := range *fed.Exports {
// Set path for object to upload/download
tempPath := tempDir
dirName := filepath.Base(tempPath)
uploadURL := fmt.Sprintf("pelican://%s/%s/%s", export.FederationPrefix, "pel_pel", dirName)

//////////////////////////////////////////////////////////
uploadURL := fmt.Sprintf("pelican://%s:%s%s/%s/%s", param.Server_Hostname.GetString(), strconv.Itoa(param.Server_WebPort.GetInt()),
export.FederationPrefix, "osdf_osdf", dirName)

// Upload the file with PUT
transferDetailsUpload, err := client.DoPut(ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
transferDetailsUpload, err := client.DoPut(fed.Ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytes17 := 0
Expand Down Expand Up @@ -221,9 +160,9 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
// Download the files we just uploaded
var transferDetailsDownload []client.TransferResults
if export.Capabilities.PublicReads {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true)
transferDetailsDownload, err = client.DoGet(fed.Ctx, uploadURL, t.TempDir(), true)
} else {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true, client.WithTokenLocation(tempToken.Name()))
transferDetailsDownload, err = client.DoGet(fed.Ctx, uploadURL, t.TempDir(), true, client.WithTokenLocation(tempToken.Name()))
}
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
Expand Down Expand Up @@ -256,19 +195,26 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
})

t.Run("testOsdfRecursiveGetAndPutOsdfURL", func(t *testing.T) {
config.SetPreferredPrefix("osdf")
config.SetPreferredPrefix("OSDF")
for _, export := range *fed.Exports {
// Set path for object to upload/download
tempPath := tempDir
dirName := filepath.Base(tempPath)
// Note: minimally fixing this test as it is soon to be replaced
uploadURL := fmt.Sprintf("pelican://%s:%s%s/%s/%s", param.Server_Hostname.GetString(), strconv.Itoa(param.Server_WebPort.GetInt()),
uploadURL := fmt.Sprintf("osdf:///%s/%s/%s",
export.FederationPrefix, "osdf_osdf", dirName)

//////////////////////////////////////////////////////////
hostname := fmt.Sprintf("%v:%v", param.Server_WebHost.GetString(), param.Server_WebPort.GetInt())

// Set our metadata values in config since that is what this url scheme - prefix combo does in handle_http
metadata, err := config.DiscoverUrlFederation("https://" + hostname)
assert.NoError(t, err)
viper.Set("Federation.DirectorUrl", metadata.DirectorEndpoint)
viper.Set("Federation.RegistryUrl", metadata.NamespaceRegistrationEndpoint)
viper.Set("Federation.DiscoveryUrl", hostname)

// Upload the file with PUT
transferDetailsUpload, err := client.DoPut(ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
transferDetailsUpload, err := client.DoPut(fed.Ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytes17 := 0
Expand Down Expand Up @@ -300,9 +246,9 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
tmpDir := t.TempDir()
var transferDetailsDownload []client.TransferResults
if export.Capabilities.PublicReads {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, tmpDir, true)
transferDetailsDownload, err = client.DoGet(fed.Ctx, uploadURL, tmpDir, true)
} else {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, tmpDir, true, client.WithTokenLocation(tempToken.Name()))
transferDetailsDownload, err = client.DoGet(fed.Ctx, uploadURL, tmpDir, true, client.WithTokenLocation(tempToken.Name()))
}
assert.NoError(t, err)
if err == nil && len(transferDetailsDownload) == 2 {
Expand Down Expand Up @@ -341,17 +287,17 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
})

t.Run("testOsdfRecursiveGetAndPutPelicanURL", func(t *testing.T) {
config.SetPreferredPrefix("osdf")
config.SetPreferredPrefix("OSDF")
for _, export := range *fed.Exports {
// Set path for object to upload/download
tempPath := tempDir
dirName := filepath.Base(tempPath)
uploadURL := fmt.Sprintf("pelican://%s/%s/%s", export.FederationPrefix, "osdf_pel", dirName)
uploadURL := fmt.Sprintf("pelican://%s:%s%s/%s/%s", param.Server_Hostname.GetString(), strconv.Itoa(param.Server_WebPort.GetInt()),
export.FederationPrefix, "osdf_osdf", dirName)

//////////////////////////////////////////////////////////

// Upload the file with PUT
transferDetailsUpload, err := client.DoPut(ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
transferDetailsUpload, err := client.DoPut(fed.Ctx, tempDir, uploadURL, true, client.WithTokenLocation(tempToken.Name()))
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
countBytes17 := 0
Expand Down Expand Up @@ -382,9 +328,9 @@ func TestRecursiveUploadsAndDownloads(t *testing.T) {
// Download the files we just uploaded
var transferDetailsDownload []client.TransferResults
if export.Capabilities.PublicReads {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true)
transferDetailsDownload, err = client.DoGet(fed.Ctx, uploadURL, t.TempDir(), true)
} else {
transferDetailsDownload, err = client.DoGet(ctx, uploadURL, t.TempDir(), true, client.WithTokenLocation(tempToken.Name()))
transferDetailsDownload, err = client.DoGet(fed.Ctx, uploadURL, t.TempDir(), true, client.WithTokenLocation(tempToken.Name()))
}
assert.NoError(t, err)
if err == nil && len(transferDetailsUpload) == 2 {
Expand Down
Loading

0 comments on commit bb8f89d

Please sign in to comment.