diff --git a/cmd/main.go b/cmd/main.go index bbabc1076..01da250fc 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -127,6 +127,29 @@ Origin: FederationPrefix: /my/prefix HttpServiceUrl: "https://example.com/testfiles" Capabilities: ["PublicReads", "Writes", "Listings"] +`) + case server_utils.OriginStorageXRoot: + fmt.Fprintf(os.Stderr, ` +Export information was not correct. +For xroot backends, specify exports via the command line using the -v flag. Example: + + -v /foo:/foo -v /bar:/bar (REQUIRED --xroot-service-url upstream-xroot-url.com:1095) + +Note that this backend type requires that the Storage Prefix (before the colon) and Federation Prefix (after the colon) match. +It also requires that the exports are configured for public reads. + +Alternatively, specify Origin.Exports in the parameters.yaml file: + + Origin: + StorageType: xroot + XRootServiceUrl: upstream-xroot-url.com:1095 + Exports: + - StoragePrefix: /foo + FederationPrefix: / + Capabilities: ["PublicReads", "Writes", "Listings"] + - StoragePrefix: /bar + FederationPrefix: /bar + Capabilities: ["PublicReads", "Writes"] `) default: fmt.Fprintf(os.Stderr, "Currently-supported origin modes include posix, https, and s3, but you provided %s.", mode) diff --git a/cmd/origin.go b/cmd/origin.go index 377d1cc20..b507e5a77 100644 --- a/cmd/origin.go +++ b/cmd/origin.go @@ -48,24 +48,6 @@ var ( Short: "Start the origin service", RunE: serveOrigin, SilenceUsage: true, - PreRun: func(cmd *cobra.Command, args []string) { - // Checking these values has to happen here and not in init() because init - // doesn't have access to the actual values passed on the command line. - if ost := viper.GetString("Origin.StorageType"); ost == "s3" { - if !viper.IsSet("Origin.S3Region") || !viper.IsSet("Origin.S3ServiceUrl") { - cmd.PrintErrln("The --region, --service-url flags or equivalent config file entries are required when the origin is launched in S3 mode.") - os.Exit(1) - } - } else if ost == "posix" { - // We specifically DON'T want the region, service-url, and url-style flags if the mode is posix - if viper.IsSet("Origin.S3Region") || viper.IsSet("Origin.S3ServiceUrl") || viper.IsSet("Origin.S3UrlStyle") { - cmd.PrintErrln("The --region, --service-url, and --url-style flags are only used when the origin is launched in S3 mode.") - } - } else { - cmd.PrintErrln(fmt.Sprintf("The --mode flag must be either 'posix' or 's3', but you provided '%s'", ost)) - os.Exit(1) - } - }, } originUiCmd = &cobra.Command{ @@ -131,7 +113,7 @@ func init() { panic(err) } - // The -v flag is used when an origin is served in POSIX mode + // The -v flag is used for passing docker-style volume mounts to the origin. originServeCmd.Flags().StringSliceP("volume", "v", []string{}, "Setting the volume to /SRC:/DEST will export the contents of /SRC as /DEST in the Pelican federation") if err := viper.BindPFlag("Origin.ExportVolumes", originServeCmd.Flags().Lookup("volume")); err != nil { panic(err) @@ -179,6 +161,14 @@ instead. // However, if you give us one, you've got to give us both. originServeCmd.MarkFlagsRequiredTogether("bucket-access-keyfile", "bucket-secret-keyfile") + // The hostname flag is used to specify the hostname of the upstream xrootd server being exported by THIS origin. + // It is NOT the same as the current origin's hostname. + originServeCmd.Flags().String("xroot-service-url", "", "When configured in xroot mode, specifies the hostname and port of the upstream xroot server "+ + "(not to be mistaken with the current server's hostname).") + if err := viper.BindPFlag("Origin.XRootServiceUrl", originServeCmd.Flags().Lookup("xroot-service-url")); err != nil { + panic(err) + } + // The port any web UI stuff will be served on originServeCmd.Flags().AddFlag(portFlag) diff --git a/config/config.go b/config/config.go index 23473bdf4..1fce8d6cd 100644 --- a/config/config.go +++ b/config/config.go @@ -1248,13 +1248,50 @@ func InitServer(ctx context.Context, currentServers ServerType) error { viper.SetDefault("Cache.Url", fmt.Sprintf("https://%v", param.Server_Hostname.GetString())) } - if viper.GetString("Origin.StorageType") == "https" { - if viper.GetString("Origin.HTTPServiceUrl") == "" { - return errors.New("Origin.HTTPServiceUrl may not be empty") - } - _, err := url.Parse(viper.GetString("Origin.HTTPServiceUrl")) - if err != nil { - return errors.Wrap(err, "unable to parse Origin.HTTPServiceUrl as a URL") + if currentServers.IsEnabled(OriginType) { + ost := param.Origin_StorageType.GetString() + switch ost { + case "posix": + viper.SetDefault("Origin.SelfTest", true) + case "https": + if param.Origin_SelfTest.GetBool() { + log.Warning("Origin.SelfTest may not be enabled when the origin is configured with non-posix backends. Turning off...") + viper.Set("Origin.SelfTest", false) + } + httpSvcUrl := param.Origin_HttpServiceUrl.GetString() + if httpSvcUrl == "" { + return errors.New("Origin.HTTPServiceUrl may not be empty when the origin is configured with an https backend") + } + _, err := url.Parse(httpSvcUrl) + if err != nil { + return errors.Wrap(err, "unable to parse Origin.HTTPServiceUrl as a URL") + } + case "xroot": + if param.Origin_SelfTest.GetBool() { + log.Warning("Origin.SelfTest may not be enabled when the origin is configured with non-posix backends. Turning off...") + viper.Set("Origin.SelfTest", false) + } + xrootSvcUrl := param.Origin_XRootServiceUrl.GetString() + if xrootSvcUrl == "" { + return errors.New("Origin.XRootServiceUrl may not be empty when the origin is configured with an xroot backend") + } + _, err := url.Parse(xrootSvcUrl) + if err != nil { + return errors.Wrap(err, "unable to parse Origin.XrootServiceUrl as a URL") + } + case "s3": + if param.Origin_SelfTest.GetBool() { + log.Warning("Origin.SelfTest may not be enabled when the origin is configured with non-posix backends. Turning off...") + viper.Set("Origin.SelfTest", false) + } + s3SvcUrl := param.Origin_S3ServiceUrl.GetString() + if s3SvcUrl == "" { + return errors.New("Origin.S3ServiceUrl may not be empty when the origin is configured with an s3 backend") + } + _, err := url.Parse(s3SvcUrl) + if err != nil { + return errors.Wrap(err, "unable to parse Origin.S3ServiceUrl as a URL") + } } } diff --git a/config/resources/defaults.yaml b/config/resources/defaults.yaml index 43ca6c892..543c635c5 100644 --- a/config/resources/defaults.yaml +++ b/config/resources/defaults.yaml @@ -69,7 +69,6 @@ Origin: EnableWrites: true EnableListings: true EnableDirectReads: false - SelfTest: true Port: 8443 SelfTestInterval: 15s Registry: diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 06cd76572..0923cd36a 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -481,7 +481,7 @@ components: ["origin"] --- name: Origin.StorageType description: |+ - The type of storage underpinning the origin. Currently supported types are "posix", "https", and "s3". + The type of storage underpinning the origin. Currently supported types are "posix", "https", "s3", and "xroot". type: string default: "posix" components: ["origin"] @@ -856,6 +856,14 @@ type: string default: none components: ["origin"] --- +name: Origin.XRootServiceUrl +description: >- + When the origin is configured to export another XRootD storage backend by setting `Origin.StorageType = xroot`, the `XRootServiceUrl` + is used as the base for `root` protocol requests and should point at the upstream XRootD server. +type: string +default: none +components: ["origin"] +--- ############################ # Local cache configs # ############################ diff --git a/param/parameters.go b/param/parameters.go index abdec0f14..1ceaa30f5 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -198,6 +198,7 @@ var ( Origin_StorageType = StringParam{"Origin.StorageType"} Origin_Url = StringParam{"Origin.Url"} Origin_XRootDPrefix = StringParam{"Origin.XRootDPrefix"} + Origin_XRootServiceUrl = StringParam{"Origin.XRootServiceUrl"} Plugin_Token = StringParam{"Plugin.Token"} Registry_DbLocation = StringParam{"Registry.DbLocation"} Registry_InstitutionsUrl = StringParam{"Registry.InstitutionsUrl"} diff --git a/param/parameters_struct.go b/param/parameters_struct.go index 7a86a7321..ea1e633c5 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -207,6 +207,7 @@ type Config struct { StorageType string Url string XRootDPrefix string + XRootServiceUrl string } Plugin struct { Token string @@ -482,6 +483,7 @@ type configWithType struct { StorageType struct { Type string; Value string } Url struct { Type string; Value string } XRootDPrefix struct { Type string; Value string } + XRootServiceUrl struct { Type string; Value string } } Plugin struct { Token struct { Type string; Value string } diff --git a/server_utils/origin.go b/server_utils/origin.go index 0687d71ee..1f06459a1 100644 --- a/server_utils/origin.go +++ b/server_utils/origin.go @@ -56,14 +56,18 @@ type ( ) var ( - ErrUnknownOriginStorageType = errors.New("unknown origin storage type") - ErrInvalidOriginConfig = errors.New("invalid origin configuration") + ErrUnknownOriginStorageType = errors.New("unknown origin storage type") + ErrInvalidOriginConfig = errors.New("invalid origin configuration") + WarnExportVolumes string = "Passing export volumes via -v at the command line causes Pelican to ignore exports configured via the yaml file. " + + "However, namespaces exported this way will inherit the Origin.Enable* settings from your configuration file. " + + "For finer-grained control of each export, please configure them in your pelican.yaml file via Origin.Exports" ) const ( OriginStoragePosix OriginStorageType = "posix" OriginStorageS3 OriginStorageType = "s3" OriginStorageHTTPS OriginStorageType = "https" + OriginStorageXRoot OriginStorageType = "xroot" // Not meant to be extensible, but facilitates legacy OSDF --> Pelican transition ) // Convert a string to an OriginStorageType @@ -75,8 +79,10 @@ func ParseOriginStorageType(storageType string) (ost OriginStorageType, err erro ost = OriginStorageHTTPS case string(OriginStoragePosix): ost = OriginStoragePosix + case string(OriginStorageXRoot): + ost = OriginStorageXRoot default: - err = errors.Wrapf(ErrUnknownOriginStorageType, "storage type %s (known types are posix, s3, and https)", storageType) + err = errors.Wrapf(ErrUnknownOriginStorageType, "storage type %s (known types are posix, s3, https, and xroot)", storageType) } return } @@ -316,9 +322,7 @@ func GetOriginExports() ([]OriginExport, error) { viper.Set("Origin.EnableReads", tmpExports[0].Capabilities.Reads) } - log.Warningln("Passing export volumes via -v at the command line causes Pelican to ignore exports configured via the yaml file") - log.Warningln("However, namespaces exported this way will inherit the Origin.Enable* settings from your configuration") - log.Warningln("For finer-grained control of each export, please configure them in your pelican.yaml file via Origin.Exports") + log.Warningln(WarnExportVolumes) originExports = tmpExports return originExports, nil } @@ -425,9 +429,7 @@ from S3 service URL. In this configuration, objects can be accessed at /federati viper.Set("Origin.EnableReads", originExports[0].Capabilities.Reads) } - log.Warningln("Passing export volumes via -v at the command line causes Pelican to ignore exports configured via the yaml file") - log.Warningln("However, namespaces exported this way will inherit the Origin.Enable* settings from your configuration") - log.Warningln("For finer-grained control of each export, please configure them in your pelican.yaml file") + log.Warningln(WarnExportVolumes) return originExports, nil } @@ -484,6 +486,115 @@ from S3 service URL. In this configuration, objects can be accessed at /federati S3SecretKeyfile: param.Origin_S3SecretKeyfile.GetString(), Capabilities: capabilities, } + viper.Set("Origin.EnableReads", capabilities.Reads) + } + case OriginStorageXRoot: + if len(param.Origin_ExportVolumes.GetStringSlice()) > 0 { + log.Infoln("Configuring exports from export volumes passed via command line or via yaml") + // This storage backend only works with unauthenticated origins. Check that now. + if !capabilities.PublicReads { + return nil, errors.Wrap(ErrInvalidOriginConfig, "the xroot backend requires that Origin.EnablePublicReads is true") + } + + volumes := param.Origin_ExportVolumes.GetStringSlice() + tmpExports := make([]OriginExport, len(volumes)) + for idx, volume := range volumes { + storagePrefix := filepath.Clean(volume) + federationPrefix := filepath.Clean(volume) + volumeMountInfo := strings.SplitN(volume, ":", 2) + if len(volumeMountInfo) == 2 { + storagePrefix = filepath.Clean(volumeMountInfo[0]) + federationPrefix = filepath.Clean(volumeMountInfo[1]) + } + + if storagePrefix != federationPrefix { + return nil, errors.Wrapf(ErrInvalidOriginConfig, "federation and storage prefixes must be the same for xroot backends, but you "+ + "provided %s and %s", storagePrefix, federationPrefix) + } + + if err = validateExportPaths(storagePrefix, federationPrefix); err != nil { + return nil, err + } + + originExport := OriginExport{ + FederationPrefix: federationPrefix, + StoragePrefix: storagePrefix, + Capabilities: capabilities, + } + tmpExports[idx] = originExport + } + + // If we're only exporting one namespace, we can set the internal Origin.FederationPrefix and Origin.StoragePrefix + if len(volumes) == 1 { + viper.Set("Origin.FederationPrefix", tmpExports[0].FederationPrefix) + viper.Set("Origin.StoragePrefix", tmpExports[0].StoragePrefix) + } + + log.Warningln(WarnExportVolumes) + originExports = tmpExports + + return originExports, nil + } + + if param.Origin_Exports.IsSet() { + log.Infoln("Configuring multi-exports from origin Exports block in config file") + var tmpExports []OriginExport + if err := viper.UnmarshalKey("Origin.Exports", &tmpExports, viper.DecodeHook(StringListToCapsHookFunc())); err != nil { + return nil, err + } + if len(tmpExports) == 0 { + err := errors.New("Origin.Exports is defined, but no exports were found") + return nil, err + } else if len(tmpExports) == 1 { + // Again, several viper variables might not be set in config. We set them here so that + // sections of code assuming a single export can make use of them. + capabilities := tmpExports[0].Capabilities + reads := capabilities.Reads || capabilities.PublicReads + viper.Set("Origin.FederationPrefix", (tmpExports)[0].FederationPrefix) + viper.Set("Origin.StoragePrefix", (tmpExports)[0].StoragePrefix) + viper.Set("Origin.EnableReads", reads) + viper.Set("Origin.EnablePublicReads", capabilities.PublicReads) + viper.Set("Origin.EnableWrites", capabilities.Writes) + viper.Set("Origin.EnableListings", capabilities.Listings) + viper.Set("Origin.EnableDirectReads", capabilities.DirectReads) + } + for _, export := range tmpExports { + if !export.Capabilities.PublicReads { + return nil, errors.Wrapf(ErrInvalidOriginConfig, "all exports from an xroot backend must have the PublicReads capability, but the export with FederationPrefix "+ + "'%s' did not", export.FederationPrefix) + } + // Paths must be the same for the XRoot backend + if export.StoragePrefix != export.FederationPrefix { + return nil, errors.Wrapf(ErrInvalidOriginConfig, "federation and storage prefixes must be the same for xroot backends, but you "+ + "provided %s and %s", export.StoragePrefix, export.FederationPrefix) + } + + if err = validateExportPaths(export.StoragePrefix, export.FederationPrefix); err != nil { + return nil, err + } + } + originExports = tmpExports + return originExports, nil + } else { + log.Infoln("Configuring single-export origin") + if !capabilities.PublicReads { + return nil, errors.Wrap(ErrInvalidOriginConfig, "the xroot backend requires the PublicReads capability, but does not have it") + } + + originExport = OriginExport{ + FederationPrefix: param.Origin_FederationPrefix.GetString(), + StoragePrefix: param.Origin_StoragePrefix.GetString(), + Capabilities: capabilities, + } + if originExport.StoragePrefix != originExport.FederationPrefix { + return nil, errors.Wrapf(ErrInvalidOriginConfig, "federation and storage prefixes must be the same for xroot backends, but you "+ + "provided %s and %s", originExport.StoragePrefix, originExport.FederationPrefix) + } + + if err = validateExportPaths(originExport.StoragePrefix, originExport.FederationPrefix); err != nil { + return nil, err + } + viper.Set("Origin.EnableReads", capabilities.Reads) } } diff --git a/xrootd/resources/xrootd-origin.cfg b/xrootd/resources/xrootd-origin.cfg index 2ca1a70aa..e6b8b4773 100644 --- a/xrootd/resources/xrootd-origin.cfg +++ b/xrootd/resources/xrootd-origin.cfg @@ -81,6 +81,12 @@ ofs.osslib libXrdHTTPServer.so httpserver.url_base {{.Origin.HttpServiceUrl}} httpserver.storage_prefix {{.Origin.FederationPrefix}} httpserver.trace debug info warning +{{else if eq .Origin.StorageType "xroot"}} +# This "origin" is actually acting like a cache that doesn't cache anything by pointing +# to another xrootd server. It allows us to plug bespoke XRootD servers into the federation +# because, after all, everything can be solved with yet another layer of indirection. +pss.origin {{.Origin.XRootServiceUrl}} +ofs.osslib libXrdPss.so {{end}} xrootd.seclib libXrdSec.so sec.protocol ztn diff --git a/xrootd/xrootd_config.go b/xrootd/xrootd_config.go index 5bfe6d88b..97ba61710 100644 --- a/xrootd/xrootd_config.go +++ b/xrootd/xrootd_config.go @@ -91,6 +91,7 @@ type ( CalculatedPort string FederationPrefix string HttpServiceUrl string + XRootServiceUrl string RunLocation string StorageType string