Merge pull request PelicanPlatform#1285 from jhiemstrawisc/issue-1261

Add an XRoot storage backend

cmd/main.go

@@ -127,6 +127,29 @@ Origin:
 	FederationPrefix: /my/prefix
 	HttpServiceUrl: "https://example.com/testfiles"
 	Capabilities: ["PublicReads", "Writes", "Listings"]
+`)
+			case server_utils.OriginStorageXRoot:
+				fmt.Fprintf(os.Stderr, `
+Export information was not correct.
+For xroot backends, specify exports via the command line using the -v flag.  Example:
+
+	-v /foo:/foo -v /bar:/bar (REQUIRED --xroot-service-url upstream-xroot-url.com:1095)
+
+Note that this backend type requires that the Storage Prefix (before the colon) and Federation Prefix (after the colon) match.
+It also requires that the exports are configured for public reads.
+
+Alternatively, specify Origin.Exports in the parameters.yaml file:
+
+	Origin:
+		StorageType: xroot
+		XRootServiceUrl: upstream-xroot-url.com:1095
+		Exports:
+		- StoragePrefix: /foo
+		  FederationPrefix: /
+		  Capabilities: ["PublicReads", "Writes", "Listings"]
+		- StoragePrefix: /bar
+		  FederationPrefix: /bar
+		  Capabilities: ["PublicReads", "Writes"]
 `)
 			default:
 				fmt.Fprintf(os.Stderr, "Currently-supported origin modes include posix, https, and s3, but you provided %s.", mode)

cmd/origin.go

@@ -48,24 +48,6 @@ var (
 		Short:        "Start the origin service",
 		RunE:         serveOrigin,
 		SilenceUsage: true,
-		PreRun: func(cmd *cobra.Command, args []string) {
-			// Checking these values has to happen here and not in init() because init
-			// doesn't have access to the actual values passed on the command line.
-			if ost := viper.GetString("Origin.StorageType"); ost == "s3" {
-				if !viper.IsSet("Origin.S3Region") || !viper.IsSet("Origin.S3ServiceUrl") {
-					cmd.PrintErrln("The --region, --service-url flags or equivalent config file entries are required when the origin is launched in S3 mode.")
-					os.Exit(1)
-				}
-			} else if ost == "posix" {
-				// We specifically DON'T want the region, service-url, and url-style flags if the mode is posix
-				if viper.IsSet("Origin.S3Region") || viper.IsSet("Origin.S3ServiceUrl") || viper.IsSet("Origin.S3UrlStyle") {
-					cmd.PrintErrln("The --region, --service-url, and --url-style flags are only used when the origin is launched in S3 mode.")
-				}
-			} else {
-				cmd.PrintErrln(fmt.Sprintf("The --mode flag must be either 'posix' or 's3', but you provided '%s'", ost))
-				os.Exit(1)
-			}
-		},
 	}
 
 	originUiCmd = &cobra.Command{
@@ -131,7 +113,7 @@ func init() {
 		panic(err)
 	}
 
-	// The -v flag is used when an origin is served in POSIX mode
+	// The -v flag is used for passing docker-style volume mounts to the origin.
 	originServeCmd.Flags().StringSliceP("volume", "v", []string{}, "Setting the volume to /SRC:/DEST will export the contents of /SRC as /DEST in the Pelican federation")
 	if err := viper.BindPFlag("Origin.ExportVolumes", originServeCmd.Flags().Lookup("volume")); err != nil {
 		panic(err)
@@ -179,6 +161,14 @@ instead.
 	// However, if you give us one, you've got to give us both.
 	originServeCmd.MarkFlagsRequiredTogether("bucket-access-keyfile", "bucket-secret-keyfile")
 
+	// The hostname flag is used to specify the hostname of the upstream xrootd server being exported by THIS origin.
+	// It is NOT the same as the current origin's hostname.
+	originServeCmd.Flags().String("xroot-service-url", "", "When configured in xroot mode, specifies the hostname and port of the upstream xroot server "+
+		"(not to be mistaken with the current server's hostname).")
+	if err := viper.BindPFlag("Origin.XRootServiceUrl", originServeCmd.Flags().Lookup("xroot-service-url")); err != nil {
+		panic(err)
+	}
+
 	// The port any web UI stuff will be served on
 	originServeCmd.Flags().AddFlag(portFlag)
 

config/config.go

@@ -1248,13 +1248,50 @@ func InitServer(ctx context.Context, currentServers ServerType) error {
 		viper.SetDefault("Cache.Url", fmt.Sprintf("https://%v", param.Server_Hostname.GetString()))
 	}
 
-	if viper.GetString("Origin.StorageType") == "https" {
-		if viper.GetString("Origin.HTTPServiceUrl") == "" {
-			return errors.New("Origin.HTTPServiceUrl may not be empty")
-		}
-		_, err := url.Parse(viper.GetString("Origin.HTTPServiceUrl"))
-		if err != nil {
-			return errors.Wrap(err, "unable to parse Origin.HTTPServiceUrl as a URL")
+	if currentServers.IsEnabled(OriginType) {
+		ost := param.Origin_StorageType.GetString()
+		switch ost {
+		case "posix":
+			viper.SetDefault("Origin.SelfTest", true)
+		case "https":
+			if param.Origin_SelfTest.GetBool() {
+				log.Warning("Origin.SelfTest may not be enabled when the origin is configured with non-posix backends. Turning off...")
+				viper.Set("Origin.SelfTest", false)
+			}
+			httpSvcUrl := param.Origin_HttpServiceUrl.GetString()
+			if httpSvcUrl == "" {
+				return errors.New("Origin.HTTPServiceUrl may not be empty when the origin is configured with an https backend")
+			}
+			_, err := url.Parse(httpSvcUrl)
+			if err != nil {
+				return errors.Wrap(err, "unable to parse Origin.HTTPServiceUrl as a URL")
+			}
+		case "xroot":
+			if param.Origin_SelfTest.GetBool() {
+				log.Warning("Origin.SelfTest may not be enabled when the origin is configured with non-posix backends. Turning off...")
+				viper.Set("Origin.SelfTest", false)
+			}
+			xrootSvcUrl := param.Origin_XRootServiceUrl.GetString()
+			if xrootSvcUrl == "" {
+				return errors.New("Origin.XRootServiceUrl may not be empty when the origin is configured with an xroot backend")
+			}
+			_, err := url.Parse(xrootSvcUrl)
+			if err != nil {
+				return errors.Wrap(err, "unable to parse Origin.XrootServiceUrl as a URL")
+			}
+		case "s3":
+			if param.Origin_SelfTest.GetBool() {
+				log.Warning("Origin.SelfTest may not be enabled when the origin is configured with non-posix backends. Turning off...")
+				viper.Set("Origin.SelfTest", false)
+			}
+			s3SvcUrl := param.Origin_S3ServiceUrl.GetString()
+			if s3SvcUrl == "" {
+				return errors.New("Origin.S3ServiceUrl may not be empty when the origin is configured with an s3 backend")
+			}
+			_, err := url.Parse(s3SvcUrl)
+			if err != nil {
+				return errors.Wrap(err, "unable to parse Origin.S3ServiceUrl as a URL")
+			}
 		}
 	}
 

config/resources/defaults.yaml

@@ -69,7 +69,6 @@ Origin:
   EnableWrites: true
   EnableListings: true
   EnableDirectReads: false
-  SelfTest: true
   Port: 8443
   SelfTestInterval: 15s
 Registry:

docs/parameters.yaml

@@ -481,7 +481,7 @@ components: ["origin"]
 ---
 name: Origin.StorageType
 description: |+
-  The type of storage underpinning the origin. Currently supported types are "posix", "https", and "s3".
+  The type of storage underpinning the origin. Currently supported types are "posix", "https", "s3", and "xroot".
 type: string
 default: "posix"
 components: ["origin"]
@@ -856,6 +856,14 @@ type: string
 default: none
 components: ["origin"]
 ---
+name: Origin.XRootServiceUrl
+description: >-
+ When the origin is configured to export another XRootD storage backend by setting `Origin.StorageType = xroot`, the `XRootServiceUrl`
+ is used as the base for `root` protocol requests and should point at the upstream XRootD server.
+type: string
+default: none
+components: ["origin"]
+---
 ############################
 #   Local cache configs    #
 ############################

server_utils/origin.go

@@ -56,14 +56,18 @@ type (
 )
 
 var (
-	ErrUnknownOriginStorageType = errors.New("unknown origin storage type")
-	ErrInvalidOriginConfig      = errors.New("invalid origin configuration")
+	ErrUnknownOriginStorageType        = errors.New("unknown origin storage type")
+	ErrInvalidOriginConfig             = errors.New("invalid origin configuration")
+	WarnExportVolumes           string = "Passing export volumes via -v at the command line causes Pelican to ignore exports configured via the yaml file. " +
+		"However, namespaces exported this way will inherit the Origin.Enable* settings from your configuration file. " +
+		"For finer-grained control of each export, please configure them in your pelican.yaml file via Origin.Exports"
 )
 
 const (
 	OriginStoragePosix OriginStorageType = "posix"
 	OriginStorageS3    OriginStorageType = "s3"
 	OriginStorageHTTPS OriginStorageType = "https"
+	OriginStorageXRoot OriginStorageType = "xroot" // Not meant to be extensible, but facilitates legacy OSDF --> Pelican transition
 )
 
 // Convert a string to an OriginStorageType
@@ -75,8 +79,10 @@ func ParseOriginStorageType(storageType string) (ost OriginStorageType, err erro
 		ost = OriginStorageHTTPS
 	case string(OriginStoragePosix):
 		ost = OriginStoragePosix
+	case string(OriginStorageXRoot):
+		ost = OriginStorageXRoot
 	default:
-		err = errors.Wrapf(ErrUnknownOriginStorageType, "storage type %s (known types are posix, s3, and https)", storageType)
+		err = errors.Wrapf(ErrUnknownOriginStorageType, "storage type %s (known types are posix, s3, https, and xroot)", storageType)
 	}
 	return
 }
@@ -316,9 +322,7 @@ func GetOriginExports() ([]OriginExport, error) {
 				viper.Set("Origin.EnableReads", tmpExports[0].Capabilities.Reads)
 			}
 
-			log.Warningln("Passing export volumes via -v at the command line causes Pelican to ignore exports configured via the yaml file")
-			log.Warningln("However, namespaces exported this way will inherit the Origin.Enable* settings from your configuration")
-			log.Warningln("For finer-grained control of each export, please configure them in your pelican.yaml file via Origin.Exports")
+			log.Warningln(WarnExportVolumes)
 			originExports = tmpExports
 			return originExports, nil
 		}
@@ -425,9 +429,7 @@ from S3 service URL. In this configuration, objects can be accessed at /federati
 				viper.Set("Origin.EnableReads", originExports[0].Capabilities.Reads)
 			}
 
-			log.Warningln("Passing export volumes via -v at the command line causes Pelican to ignore exports configured via the yaml file")
-			log.Warningln("However, namespaces exported this way will inherit the Origin.Enable* settings from your configuration")
-			log.Warningln("For finer-grained control of each export, please configure them in your pelican.yaml file")
+			log.Warningln(WarnExportVolumes)
 			return originExports, nil
 		}
 
@@ -484,6 +486,115 @@ from S3 service URL. In this configuration, objects can be accessed at /federati
 				S3SecretKeyfile:  param.Origin_S3SecretKeyfile.GetString(),
 				Capabilities:     capabilities,
 			}
+			viper.Set("Origin.EnableReads", capabilities.Reads)
+		}
+	case OriginStorageXRoot:
+		if len(param.Origin_ExportVolumes.GetStringSlice()) > 0 {
+			log.Infoln("Configuring exports from export volumes passed via command line or via yaml")
+			// This storage backend only works with unauthenticated origins. Check that now.
+			if !capabilities.PublicReads {
+				return nil, errors.Wrap(ErrInvalidOriginConfig, "the xroot backend requires that Origin.EnablePublicReads is true")
+			}
+
+			volumes := param.Origin_ExportVolumes.GetStringSlice()
+			tmpExports := make([]OriginExport, len(volumes))
+			for idx, volume := range volumes {
+				storagePrefix := filepath.Clean(volume)
+				federationPrefix := filepath.Clean(volume)
+				volumeMountInfo := strings.SplitN(volume, ":", 2)
+				if len(volumeMountInfo) == 2 {
+					storagePrefix = filepath.Clean(volumeMountInfo[0])
+					federationPrefix = filepath.Clean(volumeMountInfo[1])
+				}
+
+				if storagePrefix != federationPrefix {
+					return nil, errors.Wrapf(ErrInvalidOriginConfig, "federation and storage prefixes must be the same for xroot backends, but you "+
+						"provided %s and %s", storagePrefix, federationPrefix)
+				}
+
+				if err = validateExportPaths(storagePrefix, federationPrefix); err != nil {
+					return nil, err
+				}
+
+				originExport := OriginExport{
+					FederationPrefix: federationPrefix,
+					StoragePrefix:    storagePrefix,
+					Capabilities:     capabilities,
+				}
+				tmpExports[idx] = originExport
+			}
+
+			// If we're only exporting one namespace, we can set the internal Origin.FederationPrefix and Origin.StoragePrefix
+			if len(volumes) == 1 {
+				viper.Set("Origin.FederationPrefix", tmpExports[0].FederationPrefix)
+				viper.Set("Origin.StoragePrefix", tmpExports[0].StoragePrefix)
+			}
+
+			log.Warningln(WarnExportVolumes)
+			originExports = tmpExports
+
+			return originExports, nil
+		}
+
+		if param.Origin_Exports.IsSet() {
+			log.Infoln("Configuring multi-exports from origin Exports block in config file")
+			var tmpExports []OriginExport
+			if err := viper.UnmarshalKey("Origin.Exports", &tmpExports, viper.DecodeHook(StringListToCapsHookFunc())); err != nil {
+				return nil, err
+			}
+			if len(tmpExports) == 0 {
+				err := errors.New("Origin.Exports is defined, but no exports were found")
+				return nil, err
+			} else if len(tmpExports) == 1 {
+				// Again, several viper variables might not be set in config. We set them here so that
+				// sections of code assuming a single export can make use of them.
+				capabilities := tmpExports[0].Capabilities
+				reads := capabilities.Reads || capabilities.PublicReads
+				viper.Set("Origin.FederationPrefix", (tmpExports)[0].FederationPrefix)
+				viper.Set("Origin.StoragePrefix", (tmpExports)[0].StoragePrefix)
+				viper.Set("Origin.EnableReads", reads)
+				viper.Set("Origin.EnablePublicReads", capabilities.PublicReads)
+				viper.Set("Origin.EnableWrites", capabilities.Writes)
+				viper.Set("Origin.EnableListings", capabilities.Listings)
+				viper.Set("Origin.EnableDirectReads", capabilities.DirectReads)
+			}
+			for _, export := range tmpExports {
+				if !export.Capabilities.PublicReads {
+					return nil, errors.Wrapf(ErrInvalidOriginConfig, "all exports from an xroot backend must have the PublicReads capability, but the export with FederationPrefix "+
+						"'%s' did not", export.FederationPrefix)
+				}
+				// Paths must be the same for the XRoot backend
+				if export.StoragePrefix != export.FederationPrefix {
+					return nil, errors.Wrapf(ErrInvalidOriginConfig, "federation and storage prefixes must be the same for xroot backends, but you "+
+						"provided %s and %s", export.StoragePrefix, export.FederationPrefix)
+				}
+
+				if err = validateExportPaths(export.StoragePrefix, export.FederationPrefix); err != nil {
+					return nil, err
+				}
+			}
+			originExports = tmpExports
+			return originExports, nil
+		} else {
+			log.Infoln("Configuring single-export origin")
+			if !capabilities.PublicReads {
+				return nil, errors.Wrap(ErrInvalidOriginConfig, "the xroot backend requires the PublicReads capability, but does not have it")
+			}
+
+			originExport = OriginExport{
+				FederationPrefix: param.Origin_FederationPrefix.GetString(),
+				StoragePrefix:    param.Origin_StoragePrefix.GetString(),
+				Capabilities:     capabilities,
+			}
+			if originExport.StoragePrefix != originExport.FederationPrefix {
+				return nil, errors.Wrapf(ErrInvalidOriginConfig, "federation and storage prefixes must be the same for xroot backends, but you "+
+					"provided %s and %s", originExport.StoragePrefix, originExport.FederationPrefix)
+			}
+
+			if err = validateExportPaths(originExport.StoragePrefix, originExport.FederationPrefix); err != nil {
+				return nil, err
+			}
+
 			viper.Set("Origin.EnableReads", capabilities.Reads)
 		}
 	}

xrootd/resources/xrootd-origin.cfg

@@ -81,6 +81,12 @@ ofs.osslib libXrdHTTPServer.so
 httpserver.url_base {{.Origin.HttpServiceUrl}}
 httpserver.storage_prefix {{.Origin.FederationPrefix}}
 httpserver.trace debug info warning
+{{else if eq .Origin.StorageType "xroot"}}
+# This "origin" is actually acting like a cache that doesn't cache anything by pointing
+# to another xrootd server. It allows us to plug bespoke XRootD servers into the federation
+# because, after all, everything can be solved with yet another layer of indirection.
+pss.origin {{.Origin.XRootServiceUrl}}
+ofs.osslib libXrdPss.so
 {{end}}
 xrootd.seclib libXrdSec.so
 sec.protocol ztn

xrootd/xrootd_config.go

@@ -91,6 +91,7 @@ type (
 		CalculatedPort    string
 		FederationPrefix  string
 		HttpServiceUrl    string
+		XRootServiceUrl   string
 		RunLocation       string
 		StorageType       string