Merge branch 'main' of github.com:haoming29/pelican into fix-origin-m…

…etric-loss

.github/workflows/publish-container.yml

@@ -1,11 +1,12 @@
-
 name: Release, Build, and Push
 
 on:
   pull_request:
   push:
     tags:
-      - v[0-9]+.[0-9]+.[0-9]+
+      # only build and publish container on v7.0.0 and up
+      - v[7-9]\.[0-9]+\.[0-9]+ # match v7.x.x to v9.x.x
+      - v[1-9][0-9]+\.[0-9]+\.[0-9]+ # match any version higher
     branches:
       - main
   repository_dispatch:

.github/workflows/publish-dev-container.yml

@@ -1,25 +1,24 @@
-
 name: Release, Build, and Push Dev Image
 
 on:
   pull_request:
   push:
     tags:
-      - v[0-9]+.[0-9]+.[0-9]+-**
+      # only build and publish container on v7.0.0 and up
+      - v[7-9]\.[0-9]+\.[0-9]+-**
+      - v[1-9][0-9]+\.[0-9]+\.[0-9]+-**
     branches:
       - main
 
 jobs:
   build-dev-image:
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Checkout
+      - name: Checkout
         uses: actions/checkout@v3
         with:
-          submodules: 'recursive'
-      -
-        name: Docker meta
+          submodules: "recursive"
+      - name: Docker meta
         id: meta
         uses: docker/metadata-action@v4
         with:

.github/workflows/release.yml

@@ -4,7 +4,9 @@ name: goreleaser
 on:
   push:
     tags:
-      - '*'
+      # only run release on v7.0.0 and up
+      - v[7-9]\.[0-9]+\.[0-9]+
+      - v[1-9][0-9]+\.[0-9]+\.[0-9]+
 
 permissions:
   contents: write
@@ -13,25 +15,21 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Checkout
+      - name: Checkout
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - uses: actions/setup-node@v4
         with:
           node-version: 20
-      -
-        name: Build the website
+      - name: Build the website
         run: |
           make web-build
-      -
-        name: Set up Go
+      - name: Set up Go
         uses: actions/setup-go@v4
         with:
           go-version: "1.20"
-      -
-        name: Run GoReleaser
+      - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v4
         with:
           # either 'goreleaser' (default) or 'goreleaser-pro'

.github/workflows/test.yml

@@ -30,10 +30,21 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
+    - name: Install Mac OS X Dependencies
+      run: ./github_scripts/osx_install.sh
+      if: runner.os == 'macOS'
+    - name: Test OS X
+      if: runner.os == 'macOS'
+      run: |
+        make web-build
+        # Explicitly set XRootD to IPv4-only on Mac OS X to avoid a subtle config bug.
+        export PELICAN_XROOTD_IPV4ONLY=true
+        go test -v -coverpkg=./... -coverprofile=coverage.out -covermode=count ./...
     - name: Test
+      if: runner.os != 'macOS'
       run: |
         make web-build
-        go test -coverpkg=./... -coverprofile=coverage.out -covermode=count ./...
+        go test ./...
     - name: Run GoReleaser for Non-Ubuntu
       uses: goreleaser/goreleaser-action@v4
       with:
@@ -115,4 +126,4 @@ jobs:
       run: |
         cp dist/pelican_linux_amd64_v1/pelican ./
     - name: Run Integration Tests
-      run: ./tests/citests.sh
+      run: ./github_scripts/citests.sh

.gitignore

@@ -1,5 +1,3 @@
 dist/
 README.dev.md
-param/parameters.go
 docs/parameters.json
-param/parameters_struct.go

.goreleaser.yml

@@ -122,6 +122,7 @@ nfpms:
       # - osdf-client = {{ .Version }}
       # - stashcp = {{ .Version }}
       # - condor-stash-plugin = {{ .Version }}
+      - "stashcache-client = 7"
       - "osdf-client = 7"
       - "stashcp = 7"
       - "condor-stash-plugin = 7"
@@ -149,6 +150,7 @@ nfpms:
             dst: "/etc/condor/config.d/10-stash-plugin.conf"
             type: config|noreplace
         replaces:
+          - "stashcache-client < 7"
           - "osdf-client < 7"
           - "stashcp < 7"
           - "condor-stash-plugin < 7"
@@ -174,10 +176,12 @@ nfpms:
             type: config|noreplace
         # deb has different syntax
         provides:
+          - "stashcache-client (= 7)"
           - "osdf-client (= 7)"
           - "stashcp (= 7)"
           - "condor-stash-plugin (= 7)"
         replaces:
+          - "stashcache-client (<< 7)"
           - "osdf-client (<< 7)"
           - "stashcp (<< 7)"
           - "condor-stash-plugin (<< 7)"

.pre-commit-config.yaml

@@ -12,6 +12,6 @@ repos:
         args: [--allow-multiple-documents]
     -   id: check-added-large-files
 -   repo: https://github.com/golangci/golangci-lint
-    rev: v1.54.2
+    rev: v1.55.2
     hooks:
     -   id: golangci-lint

README.md

@@ -6,6 +6,10 @@ The Pelican command line tool allows one to use a Pelican
 federation as a client and serve datasets through running a
 Pelican origin service.
 
+For more information on Pelican, see the [Pelican Platform page](https://pelicanplatform.org/).
+
+For documentation on using the Pelican Platform, see the [Pelican Platform documentation page](https://docs.pelicanplatform.org/).
+
 Testing and Usage
 -----------------
 

client/handle_http.go

@@ -19,6 +19,7 @@
 package client
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -48,12 +49,26 @@ import (
 	"github.com/pelicanplatform/pelican/param"
 )
 
-var progressContainer = mpb.New()
+var (
+	progressCtrOnce sync.Once
+	progressCtr     *mpb.Progress
+)
 
 type StoppedTransferError struct {
 	Err string
 }
 
+// The progress container object creates several
+// background goroutines.  Instead of creating the object
+// globally, create it on first use.  This avoids having
+// the progress container routines launch in the server.
+func getProgressContainer() *mpb.Progress {
+	progressCtrOnce.Do(func() {
+		progressCtr = mpb.New()
+	})
+	return progressCtr
+}
+
 func (e *StoppedTransferError) Error() string {
 	return e.Err
 }
@@ -333,7 +348,7 @@ func download_http(sourceUrl *url.URL, destination string, payload *payloadStruc
 	//tf := TransferFiles{files: files}
 
 	if ObjectClientOptions.Recursive && ObjectClientOptions.ProgressBars {
-		log.SetOutput(progressContainer)
+		log.SetOutput(getProgressContainer())
 	}
 	// Start the workers
 	for i := 1; i <= 5; i++ {
@@ -367,7 +382,7 @@ func download_http(sourceUrl *url.URL, destination string, payload *payloadStruc
 	}
 	// Make sure to close the progressContainer after all download complete
 	if ObjectClientOptions.Recursive && ObjectClientOptions.ProgressBars {
-		progressContainer.Wait()
+		getProgressContainer().Wait()
 		log.SetOutput(os.Stdout)
 	}
 	return downloaded, downloadError
@@ -545,7 +560,7 @@ func DownloadHTTP(transfer TransferDetails, dest string, token string) (int64, e
 
 	var progressBar *mpb.Bar
 	if ObjectClientOptions.ProgressBars {
-		progressBar = progressContainer.AddBar(0,
+		progressBar = getProgressContainer().AddBar(0,
 			mpb.PrependDecorators(
 				decor.Name(filename, decor.WCSyncSpaceR),
 				decor.CountersKibiByte("% .2f / % .2f"),
@@ -610,7 +625,7 @@ Loop:
 					continue
 				} else if startBelowLimit == 0 {
 					warning := []byte("Warning! Downloading too slow...\n")
-					status, err := progressContainer.Write(warning)
+					status, err := getProgressContainer().Write(warning)
 					if err != nil {
 						log.Errorln("Problem displaying slow message", err, status)
 						continue
@@ -657,7 +672,7 @@ Loop:
 					if ObjectClientOptions.Recursive {
 						progressBar.Wait()
 					} else { // Otherwise just close it
-						progressContainer.Wait()
+						getProgressContainer().Wait()
 					}
 				}
 			}
@@ -768,7 +783,7 @@ func UploadDirectory(src string, dest *url.URL, token string, namespace namespac
 	}
 
 	if ObjectClientOptions.ProgressBars {
-		log.SetOutput(progressContainer)
+		log.SetOutput(getProgressContainer())
 	}
 	// Upload all of our files within the proper directories
 	for _, file := range files {
@@ -785,7 +800,7 @@ func UploadDirectory(src string, dest *url.URL, token string, namespace namespac
 	}
 	// Close progress bar container
 	if ObjectClientOptions.ProgressBars {
-		progressContainer.Wait()
+		getProgressContainer().Wait()
 		log.SetOutput(os.Stdout)
 	}
 	return amountDownloaded, err
@@ -834,16 +849,40 @@ func UploadFile(src string, origDest *url.URL, token string, namespace namespace
 		nonZeroSize = fileInfo.Size() > 0
 	}
 
-	// Parse the writeback host as a URL
-	writebackhostUrl, err := url.Parse(namespace.WriteBackHost)
+	// call a GET on the director, director will respond with our endpoint
+	directorUrlStr := param.Federation_DirectorUrl.GetString()
+	directorUrl, err := url.Parse(directorUrlStr)
 	if err != nil {
-		return 0, err
+		return 0, errors.Wrap(err, "failed to parse director url")
+	}
+	directorUrl.Path, err = url.JoinPath("/api/v1.0/director/origin", origDest.Path)
+	if err != nil {
+		return 0, errors.Wrap(err, "failed to parse director path for upload")
 	}
 
-	dest := &url.URL{
-		Host:   writebackhostUrl.Host,
-		Scheme: "https",
-		Path:   origDest.Path,
+	payload := []byte("forPUT")
+	req, err := http.NewRequest("GET", directorUrl.String(), bytes.NewBuffer(payload))
+	if err != nil {
+		return 0, errors.Wrap(err, "failed to construct request for director-origin query")
+	}
+
+	client := &http.Client{
+		Transport: config.GetTransport(),
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, errors.Wrap(err, "failed to send request to director to obtain upload endpoint")
+	}
+	if resp.StatusCode == 405 {
+		return 0, errors.New("Error 405: No writeable origins were found")
+	}
+	defer resp.Body.Close()
+	dest, err := url.Parse(resp.Header.Get("Location"))
+	if err != nil {
+		return 0, errors.Wrap(err, "failed to parse location header from director response")
 	}
 
 	// Create the wrapped reader and send it to the request
@@ -875,7 +914,7 @@ func UploadFile(src string, origDest *url.URL, token string, namespace namespace
 
 	var progressBar *mpb.Bar
 	if ObjectClientOptions.ProgressBars {
-		progressBar = progressContainer.AddBar(0,
+		progressBar = getProgressContainer().AddBar(0,
 			mpb.PrependDecorators(
 				decor.Name(src, decor.WCSyncSpaceR),
 				decor.CountersKibiByte("% .2f / % .2f"),
@@ -897,7 +936,7 @@ func UploadFile(src string, origDest *url.URL, token string, namespace namespace
 			if ObjectClientOptions.Recursive {
 				progressBar.Wait()
 			} else { // If not recursive, go ahead and close it
-				progressContainer.Wait()
+				getProgressContainer().Wait()
 			}
 		}()
 	}
@@ -958,10 +997,9 @@ Loop:
 
 }
 
-var UploadClient = &http.Client{Transport: config.GetTransport()}
-
 // Actually perform the Put request to the server
 func doPut(request *http.Request, responseChan chan<- *http.Response, errorChan chan<- error) {
+	var UploadClient = &http.Client{Transport: config.GetTransport()}
 	client := UploadClient
 	dump, _ := httputil.DumpRequestOut(request, false)
 	log.Debugf("Dumping request: %s", dump)