Unit test for recursive uploads/downloads & bugfix

Added some basic unit tests for recursive uploads/downloads, planning on
expanding them later but want to try to get this in for release. Also
fixed bugs with recursive uploads and the funky renaming of directories
it was doing.

client/handle_http.go

@@ -834,10 +834,30 @@ func UploadDirectory(src string, dest *url.URL, token string, namespace namespac
 		log.SetOutput(getProgressContainer())
 	}
 	var transfer TransferResults
+
+	// Check if there is a directory specified in dest, if not then use the local one
+	cleanDest := path.Clean(dest.Path)
+	destComponents := strings.Split(cleanDest, "/")
+	destDirSpecified := false
+
+	if len(destComponents) == 2 {
+		log.Debugln("No directory specified in destination, using local directory name")
+	} else {
+		destDirSpecified = true
+	}
+
 	// Upload all of our files within the proper directories
 	for _, file := range files {
 		tempDest := url.URL{}
-		tempDest.Path, err = url.JoinPath(dest.Path, file)
+
+		if destDirSpecified {
+			destFile := strings.TrimPrefix(file, src)
+			tempDest.Path, err = url.JoinPath(dest.Path, destFile)
+		} else {
+			destFile := strings.TrimPrefix(file, src)
+			tempDest.Path, err = url.JoinPath(dest.Path, path.Base(src), destFile)
+		}
+
 		if err != nil {
 			return nil, err
 		}
@@ -1120,12 +1140,6 @@ func walkDavDir(url *url.URL, namespace namespaces.Namespace, token string, dest
 func walkDirUpload(path string, client *gowebdav.Client, destPath string) ([]string, error) {
 	// List of files to return
 	var files []string
-	// Whenever this function is called, we should create a new dir on the server side for uploads
-	err := client.Mkdir(destPath+path, 0755)
-	if err != nil {
-		return nil, err
-	}
-	log.Debugf("Creating directory: %s", destPath+path)
 
 	// Get our list of files
 	infos, err := os.ReadDir(path)
@@ -1134,9 +1148,10 @@ func walkDirUpload(path string, client *gowebdav.Client, destPath string) ([]str
 	}
 	for _, info := range infos {
 		newPath := path + "/" + info.Name()
+		newDestPath := destPath + "/" + info.Name() //TODO make path.Join
 		if info.IsDir() {
 			// Recursively call this function to create any nested dir's as well as list their files
-			returnedFiles, err := walkDirUpload(newPath, client, destPath)
+			returnedFiles, err := walkDirUpload(newPath, client, newDestPath)
 			if err != nil {
 				return nil, err
 			}

cmd/object_get_put_test.go

@@ -249,3 +249,89 @@ func TestGetPublicRead(t *testing.T) {
 		assert.Contains(t, string(out), "Downloaded bytes: 17")
 	})
 }
+
+func TestRecursiveUploadsAndDownloads(t *testing.T) {
+	// Create instance of test federation
+	viper.Reset()
+	fed := FedTest{T: t}
+	fed.Spinup()
+	defer fed.Teardown()
+	t.Run("testRecursiveGetAndPut", func(t *testing.T) {
+		//////////////////////////SETUP///////////////////////////
+		// Create a token file
+		tokenConfig := utils.TokenConfig{
+			TokenProfile: utils.WLCG,
+			Lifetime:     time.Minute,
+			Issuer:       param.Origin_Url.GetString(),
+			Audience:     []string{param.Origin_Url.GetString()},
+			Subject:      "origin",
+		}
+		tokenConfig.AddRawScope("storage.read:/ storage.modify:/")
+		token, err := tokenConfig.CreateToken()
+		assert.NoError(t, err)
+		tempToken, err := os.CreateTemp(t.TempDir(), "token")
+		assert.NoError(t, err, "Error creating temp token file")
+		defer os.Remove(tempToken.Name())
+		_, err = tempToken.WriteString(token)
+		assert.NoError(t, err, "Error writing to temp token file")
+		tempToken.Close()
+
+		// Disable progress bars to not reuse the same mpb instance
+		viper.Set("Logging.DisableProgressBars", true)
+
+		// Make our test directories and files
+		tempDir, err := os.MkdirTemp("", "UploadDir")
+		assert.NoError(t, err)
+		defer os.RemoveAll(tempDir)
+		permissions := os.FileMode(0777)
+		err = os.Chmod(tempDir, permissions)
+		require.NoError(t, err)
+
+		testFileContent1 := "test file content"
+		testFileContent2 := "more test file content!"
+		tempFile1, err := os.CreateTemp(tempDir, "test1")
+		assert.NoError(t, err, "Error creating temp1 file")
+		tempFile2, err := os.CreateTemp(tempDir, "test1")
+		assert.NoError(t, err, "Error creating temp2 file")
+		defer os.Remove(tempFile1.Name())
+		defer os.Remove(tempFile2.Name())
+		_, err = tempFile1.WriteString(testFileContent1)
+		assert.NoError(t, err, "Error writing to temp1 file")
+		tempFile1.Close()
+		_, err = tempFile2.WriteString(testFileContent2)
+		assert.NoError(t, err, "Error writing to temp2 file")
+		tempFile2.Close()
+
+		// Set path for object to upload/download
+		tempPath := tempDir
+		dirName := filepath.Base(tempPath)
+		uploadURL := "osdf:///test/" + dirName
+
+		//////////////////////////////////////////////////////////
+
+		// Upload the file with PUT
+		rootCmd.SetArgs([]string{"object", "put", tempDir, uploadURL, "-d", "-r", "-t", tempToken.Name()})
+
+		err = rootCmd.Execute()
+		assert.NoError(t, err, "Failed to run pelican object put")
+		out, err := io.ReadAll(fed.Output)
+		assert.NoError(t, err)
+
+		// Confirm we're uploading size we are expecting
+		assert.Contains(t, string(out), "Uploaded bytes: 23")
+		assert.Contains(t, string(out), "Uploaded bytes: 17")
+
+		// Download that same file with GET
+		rootCmd.SetArgs([]string{"object", "get", uploadURL, t.TempDir(), "-d", "-r", "-c", param.Origin_Url.GetString(), "-t", tempToken.Name()})
+
+		err = rootCmd.Execute()
+		assert.NoError(t, err, "Failed to run pelican object get")
+
+		out, err = io.ReadAll(fed.Output)
+		assert.NoError(t, err)
+		// Confirm we download same amount of bytes as upload
+		assert.Contains(t, string(out), "Downloaded bytes: 23")
+		assert.Contains(t, string(out), "Downloaded bytes: 17")
+	})
+
+}