Implement recursive uploads

First draft of recursive uploads. This will go through a directory and
upload every file to the destination. The only problem is that it will
not maintain the same directory structure since I could not find a way
to have PUT work with directories.

client/handle_http.go

@@ -23,12 +23,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -683,6 +685,36 @@ func (pr *ProgressReader) Close() error {
 	return err
 }
 
+func UploadDirectory(src string, dest *url.URL, token string, namespace namespaces.Namespace) (int64, error) {
+	var amountUploaded int64
+	err := filepath.WalkDir(src, func(path string, info fs.DirEntry, err error) error {
+		// For now, ignore directories since you cannot PUT a dir
+		if !(info.IsDir()) {
+			// Do all this work to update the destination to include the current file at the end
+			currentDest, err := url.Parse(dest.Path)
+			if err != nil {
+				return err
+			}
+			baseDir, _ := filepath.Split(currentDest.Path)
+			newFilename := info.Name()
+			updatedPath := filepath.Join(baseDir, newFilename)
+			currentDest.Path = updatedPath
+			// Upload the file with the updated destination
+			var uploaded int64
+			uploaded, err = UploadFile(path, currentDest, token, namespace)
+			if err != nil {
+				return err
+			}
+			amountUploaded += uploaded
+		}
+		return err
+	})
+	if err != nil {
+		return 0, err
+	}
+	return amountUploaded, err
+}
+
 // UploadFile Uploads a file using HTTP
 func UploadFile(src string, dest *url.URL, token string, namespace namespaces.Namespace) (int64, error) {
 

client/main.go

@@ -103,14 +103,17 @@ func getTokenName(destination *url.URL) (scheme, tokenName string) {
 }
 
 // Do writeback to stash using SciTokens
-func doWriteBack(source string, destination *url.URL, namespace namespaces.Namespace) (int64, error) {
+func doWriteBack(source string, destination *url.URL, namespace namespaces.Namespace, recursive bool) (int64, error) {
 
 	scitoken_contents, err := getToken(destination, namespace, true, "")
 	if err != nil {
 		return 0, err
 	}
-	return UploadFile(source, destination, scitoken_contents, namespace)
-
+	if recursive {
+		return UploadDirectory(source, destination, scitoken_contents, namespace)
+	} else {
+		return UploadFile(source, destination, scitoken_contents, namespace)
+	}
 }
 
 // getToken returns the token to use for the given destination
@@ -481,7 +484,7 @@ func DoStashCPSingle(sourceFile string, destination string, methods []string, re
 			AddError(err)
 			return 0, err
 		}
-		_, err = doWriteBack(source_url.Path, dest_url, ns)
+		_, err = doWriteBack(source_url.Path, dest_url, ns, recursive)
 		AddError(err)
 		return 0, err
 	}