Merge branch 'main' of github.com:PelicanPlatform/pelican into cache-…

…watermarks
PelicanPlatform · Apr 1, 2024 · c8101dd · c8101dd
2 parents 4e37fe2 + 5635a1e
commit c8101dd
Show file tree

Hide file tree

Showing 108 changed files with 4,850 additions and 3,148 deletions.
diff --git a/.github/workflows/go-generate-check.yml b/.github/workflows/go-generate-check.yml
@@ -0,0 +1,27 @@
+name: Check Go Generate Has been Run
+on:
+  pull_request:
+jobs:
+  validate-generated-files:
+    name: Check Go Generate
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.21"
+
+      - name: Run Go Generate
+        run: go generate ./...
+
+      - name: Validate Generated Files are up to date
+        run: |
+          if git diff --quiet; then
+            echo "No changes found."
+          else
+            echo "Changes detected. Here are the details:"
+            git diff
+            exit 1
+          fi
diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
@@ -18,10 +18,18 @@ jobs:
     outputs:
       tag: ${{ steps.extract_tag.outputs.tag }}
     steps:
+
+      - name: Set Branch
+        id: set_branch
+        run: |
+          echo "branch=${GITHUB_REF_NAME%.*}.x" >> $GITHUB_OUTPUT
+
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ steps.set_branch.outputs.branch }}
+
       - uses: actions/setup-node@v4
         with:
           node-version: 20
@@ -36,11 +44,6 @@ jobs:
           # Add some verbosity
           echo "NPM version is now $(npm version)"
 
-      - name: Set Branch
-        id: set_branch
-        run: |
-          echo "branch=${GITHUB_REF_NAME%.*}.x" >> $GITHUB_OUTPUT
-
       - uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: "Pre Release: Update npm version"
@@ -64,12 +67,12 @@ jobs:
 
   release-container:
     needs: [pre-release]
-    uses: PelicanPlatform/pelican/.github/workflows/build-prod-container.yml@main
+    uses: ./.github/workflows/build-prod-container.yml
     secrets: inherit
     with:
       tag: ${{ needs.pre-release.outputs.tag }}
 
   release:
     needs: [pre-release]
-    uses: PelicanPlatform/pelican/.github/workflows/release.yml@main
+    uses: ./.github/workflows/release.yml
     secrets: inherit
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -23,9 +23,6 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 20
-      - name: Build the website
-        run: |
-          make web-build
       - name: Set up Go
         uses: actions/setup-go@v5
         with:

diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -24,6 +24,7 @@ before:
   hooks:
     - go mod tidy
     - go generate ./...
+    - make web-build
 builds:
   - env:
       - CGO_ENABLED=0

diff --git a/broker/broker_test.go b/broker/broker_test.go
@@ -39,10 +39,10 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/pelicanplatform/pelican/common"
 	"github.com/pelicanplatform/pelican/config"
 	"github.com/pelicanplatform/pelican/param"
 	"github.com/pelicanplatform/pelican/registry"
+	"github.com/pelicanplatform/pelican/server_structs"
 	"github.com/pelicanplatform/pelican/server_utils"
 	"github.com/pelicanplatform/pelican/test_utils"
 	"github.com/pelicanplatform/pelican/token_scopes"
@@ -309,7 +309,7 @@ func TestRetrieveTimeout(t *testing.T) {
 	err = json.Unmarshal(responseBytes, &brokerResp)
 	require.NoError(t, err)
 
-	assert.Equal(t, common.RespPollTimeout, brokerResp.Status)
+	assert.Equal(t, server_structs.RespPollTimeout, brokerResp.Status)
 
 	ctx, cancelFunc := context.WithTimeout(ctx, 50*time.Millisecond)
 	defer cancelFunc()

diff --git a/broker/client.go b/broker/client.go
@@ -43,9 +43,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/pelicanplatform/pelican/common"
 	"github.com/pelicanplatform/pelican/config"
 	"github.com/pelicanplatform/pelican/param"
+	"github.com/pelicanplatform/pelican/server_structs"
 	"github.com/pelicanplatform/pelican/token_scopes"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -240,7 +240,7 @@ func ConnectToOrigin(ctx context.Context, brokerUrl, prefix, originName string)
 		err = errors.Wrap(err, "Failure when reading response from broker response")
 	}
 	if resp.StatusCode >= 400 {
-		errResp := common.SimpleApiResp{}
+		errResp := server_structs.SimpleApiResp{}
 		log.Errorf("Failure (status code %d) when invoking the broker: %s", resp.StatusCode, string(responseBytes))
 		if err = json.Unmarshal(responseBytes, &errResp); err != nil {
 			err = errors.Errorf("Failure when invoking the broker (status code %d); unable to parse error message", resp.StatusCode)
@@ -299,7 +299,7 @@ func ConnectToOrigin(ctx context.Context, brokerUrl, prefix, originName string)
 		hj, ok := writer.(http.Hijacker)
 		if !ok {
 			log.Debug("Not able to hijack underlying TCP connection from server")
-			resp := common.SimpleApiResp{
+			resp := server_structs.SimpleApiResp{
 				Msg:    "Unable to reverse TCP connection; HTTP/2 in use",
 				Status: "error",
 			}
@@ -470,7 +470,7 @@ func doCallback(ctx context.Context, brokerResp reversalRequest) (listener net.L
 	}
 
 	if resp.StatusCode >= 400 {
-		errResp := common.SimpleApiResp{}
+		errResp := server_structs.SimpleApiResp{}
 		if err = json.Unmarshal(responseBytes, &errResp); err != nil {
 			err = errors.Errorf("Failure when invoking cache %s callback (status code %d); unable to parse error message", brokerResp.CallbackUrl, resp.StatusCode)
 		} else {
@@ -617,7 +617,7 @@ func LaunchRequestMonitor(ctx context.Context, egrp *errgroup.Group, resultChan
 					break
 				}
 				if resp.StatusCode >= 400 {
-					errResp := common.SimpleApiResp{}
+					errResp := server_structs.SimpleApiResp{}
 					if err = json.Unmarshal(responseBytes, &errResp); err != nil {
 						log.Errorf("Failure when invoking the broker (status code %d); unable to parse error message", resp.StatusCode)
 					} else {
@@ -634,17 +634,17 @@ func LaunchRequestMonitor(ctx context.Context, egrp *errgroup.Group, resultChan
 					break
 				}
 
-				if brokerResp.Status == common.RespOK {
+				if brokerResp.Status == server_structs.RespOK {
 					listener, err := doCallback(ctx, brokerResp.Request)
 					if err != nil {
 						log.Errorln("Failed to callback to the cache:", err)
 						resultChan <- err
 						break
 					}
 					resultChan <- listener
-				} else if brokerResp.Status == common.RespFailed {
+				} else if brokerResp.Status == server_structs.RespFailed {
 					log.Errorln("Broker responded to origin retrieve with an error:", brokerResp.Msg)
-				} else if brokerResp.Status != common.RespPollTimeout { // We expect timeouts; do not log them.
+				} else if brokerResp.Status != server_structs.RespPollTimeout { // We expect timeouts; do not log them.
 					if brokerResp.Msg != "" {
 						log.Errorf("Broker responded with unknown status (%s); msg: %s", brokerResp.Status, brokerResp.Msg)
 					} else {

diff --git a/broker/server_apis.go b/broker/server_apis.go
@@ -25,8 +25,8 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/pelicanplatform/pelican/common"
 	"github.com/pelicanplatform/pelican/param"
+	"github.com/pelicanplatform/pelican/server_structs"
 	"github.com/pelicanplatform/pelican/token_scopes"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -42,7 +42,7 @@ type (
 
 	// Response for a successful retrieval
 	brokerRetrievalResp struct {
-		common.SimpleApiResp
+		server_structs.SimpleApiResp
 		Request reversalRequest `json:"req"`
 	}
 
@@ -54,19 +54,19 @@ type (
 
 func newBrokerReqResp(req reversalRequest) (result brokerRetrievalResp) {
 	result.Request = req
-	result.SimpleApiResp.Status = common.RespOK
+	result.SimpleApiResp.Status = server_structs.RespOK
 	return
 }
 
-func newBrokerRespFail(msg string) common.SimpleApiResp {
-	return common.SimpleApiResp{
-		Status: common.RespFailed,
+func newBrokerRespFail(msg string) server_structs.SimpleApiResp {
+	return server_structs.SimpleApiResp{
+		Status: server_structs.RespFailed,
 		Msg:    msg,
 	}
 }
 
 func newBrokerRespTimeout() (result brokerRetrievalResp) {
-	result.SimpleApiResp.Status = common.RespPollTimeout
+	result.SimpleApiResp.Status = server_structs.RespPollTimeout
 	return
 }
 

diff --git a/cache_ui/advertise.go → cache/advertise.go b/cache_ui/advertise.go → cache/advertise.go
@@ -16,32 +16,30 @@
  *
  ***************************************************************/
 
-package cache_ui
+package cache
 
 import (
 	"context"
 	"encoding/json"
 	"net/url"
 	"strings"
 
-	"github.com/pelicanplatform/pelican/common"
 	"github.com/pelicanplatform/pelican/config"
-	"github.com/pelicanplatform/pelican/director"
 	"github.com/pelicanplatform/pelican/param"
-	"github.com/pelicanplatform/pelican/server_utils"
+	"github.com/pelicanplatform/pelican/server_structs"
 	"github.com/pelicanplatform/pelican/utils"
 	"github.com/pkg/errors"
 )
 
 type (
 	CacheServer struct {
-		server_utils.NamespaceHolder
+		server_structs.NamespaceHolder
 		namespaceFilter map[string]struct{}
 	}
 )
 
-func (server *CacheServer) CreateAdvertisement(name string, originUrl string, originWebUrl string) (*common.OriginAdvertiseV2, error) {
-	ad := common.OriginAdvertiseV2{
+func (server *CacheServer) CreateAdvertisement(name string, originUrl string, originWebUrl string) (*server_structs.OriginAdvertiseV2, error) {
+	ad := server_structs.OriginAdvertiseV2{
 		Name:       name,
 		DataURL:    originUrl,
 		WebURL:     originWebUrl,
@@ -68,13 +66,13 @@ func (server *CacheServer) SetFilters() {
 	}
 }
 
-func (server *CacheServer) filterAdsBasedOnNamespace(nsAds []common.NamespaceAdV2) []common.NamespaceAdV2 {
+func (server *CacheServer) filterAdsBasedOnNamespace(nsAds []server_structs.NamespaceAdV2) []server_structs.NamespaceAdV2 {
 	/*
 	* Filters out ads based on the namespaces listed in server.NamespaceFilter
 	* Note that this does a few checks for trailing and non-trailing "/" as it's assumed that the namespaces
 	* from the director and the ones provided might differ.
 	 */
-	filteredAds := []common.NamespaceAdV2{}
+	filteredAds := []server_structs.NamespaceAdV2{}
 	if len(server.namespaceFilter) > 0 {
 		for _, ad := range nsAds {
 			ns := ad.Path
@@ -111,7 +109,7 @@ func (server *CacheServer) filterAdsBasedOnNamespace(nsAds []common.NamespaceAdV
 
 func (server *CacheServer) GetNamespaceAdsFromDirector() error {
 	// Get the endpoint of the director
-	var respNS []common.NamespaceAdV2
+	var respNS []server_structs.NamespaceAdV2
 
 	directorEndpoint := param.Federation_DirectorUrl.GetString()
 	if directorEndpoint == "" {
@@ -140,14 +138,14 @@ func (server *CacheServer) GetNamespaceAdsFromDirector() error {
 				return err
 			}
 			respData, err = utils.MakeRequest(context.Background(), directorNSListEndpointURL, "GET", nil, nil)
-			var respNSV1 []common.NamespaceAdV1
+			var respNSV1 []server_structs.NamespaceAdV1
 			if err != nil {
 				return errors.Wrap(err, "Failed to make request")
 			} else {
 				if jsonErr := json.Unmarshal(respData, &respNSV1); jsonErr == nil { // Error creating json
 					return errors.Wrapf(err, "Failed to make request: %v", err)
 				}
-				respNS = director.ConvertNamespaceAdsV1ToV2(respNSV1, nil)
+				respNS = server_structs.ConvertNamespaceAdsV1ToV2(respNSV1, nil)
 			}
 		} else {
 			return errors.Wrap(err, "Failed to make request")

diff --git a/cache_ui/advertise_test.go → cache/advertise_test.go b/cache_ui/advertise_test.go → cache/advertise_test.go
@@ -16,15 +16,15 @@
  *
  ***************************************************************/
 
-package cache_ui
+package cache
 
 import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
-	"github.com/pelicanplatform/pelican/common"
+	"github.com/pelicanplatform/pelican/server_structs"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 )
@@ -83,7 +83,7 @@ func TestFilterNsAdsForCache(t *testing.T) {
 	viper.Reset()
 	defer viper.Reset()
 
-	nsAds := []common.NamespaceAdV2{
+	nsAds := []server_structs.NamespaceAdV2{
 		{
 			Path: "/ns1",
 		},

diff --git a/cache_ui/broker_client.go → cache/broker_client.go b/cache_ui/broker_client.go → cache/broker_client.go
@@ -18,7 +18,7 @@
  *
  ***************************************************************/
 
-package cache_ui
+package cache
 
 import (
 	"bufio"

diff --git a/cache_ui/broker_client_windows.go → cache/broker_client_windows.go b/cache_ui/broker_client_windows.go → cache/broker_client_windows.go
@@ -18,7 +18,7 @@
  *
  ***************************************************************/
 
-package cache_ui
+package cache
 
 import (
 	"context"