From e40950e3b876bc84bffc0268a59dfd5f8ab0ded8 Mon Sep 17 00:00:00 2001 From: Justin Hiemstra Date: Thu, 29 Aug 2024 18:16:41 +0000 Subject: [PATCH] Fix topology downtime integration by using downtime URL and not ns JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This changes the overall approach for grabbing downtime information from Topology. Previously, we generated the list of downed servers by getting a diff of the servers returned in the namespaces JSON when querying that endpoint with `?include_downed=<0/1>`. The issue with this approach is that Topology services marked as Pelican services are explicitly excluded from the returned JSON. However, Pelican services that are recorded in Topology are still included in the downtime URL, so this new approach grabs the list of resource names from that instead. That was probably the right approach all along, but topology endpoints remain inscrutable ¯\_(ツ)_/¯ This also corrects a documentation issue that claimed filtered servers could be added in the Director using their hostname. In fact, they need to be filtered using their site name! --- config/resources/osdf.yaml | 5 +- director/advertise.go | 85 ++++--- director/advertise_test.go | 210 ++++++------------ .../mock_topology_downtime_template.xml | 30 +++ docs/parameters.yaml | 12 +- param/parameters.go | 1 + param/parameters_struct.go | 2 + registry/registry_db.go | 2 +- server_structs/topology.go | 84 +++++++ server_utils/server_utils.go | 48 +--- 10 files changed, 256 insertions(+), 223 deletions(-) create mode 100644 director/resources/mock_topology_downtime_template.xml create mode 100644 server_structs/topology.go diff --git a/config/resources/osdf.yaml b/config/resources/osdf.yaml index de9c88037..47f7ed8cb 100644 --- a/config/resources/osdf.yaml +++ b/config/resources/osdf.yaml @@ -20,8 +20,9 @@ Xrootd: DetailedMonitoringHost: xrd-mon.osgstorage.org Federation: DiscoveryUrl: osg-htc.org - TopologyURL: https://topology.opensciencegrid.org - TopologyNamespaceURL: https://topology.opensciencegrid.org/osdf/namespaces?production=1 + TopologyUrl: https://topology.opensciencegrid.org + TopologyNamespaceUrl: https://topology.opensciencegrid.org/osdf/namespaces?production=1 + TopologyDowntimeUrl: https://topology.opensciencegrid.org/rgdowntime/xml TopologyReloadInterval: 4.5m Registry: RequireCacheApproval: true diff --git a/director/advertise.go b/director/advertise.go index 80b9f6a35..2bf532b51 100644 --- a/director/advertise.go +++ b/director/advertise.go @@ -20,6 +20,8 @@ package director import ( "context" + "encoding/xml" + "net/http" "net/url" "strings" "time" @@ -27,9 +29,11 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" + "github.com/pelicanplatform/pelican/utils" ) // Consolite two ServerAds that share the same ServerAd.URL. For all but the capability fields, @@ -54,7 +58,7 @@ func consolidateDupServerAd(newAd, existingAd server_structs.ServerAd) server_st // Takes in server information from topology and handles converting the necessary bits into a new Pelican // ServerAd. -func parseServerAdFromTopology(server server_utils.Server, serverType server_structs.ServerType, caps server_structs.Capabilities) server_structs.ServerAd { +func parseServerAdFromTopology(server server_structs.TopoServer, serverType server_structs.ServerType, caps server_structs.Capabilities) server_structs.ServerAd { serverAd := server_structs.ServerAd{} serverAd.Type = serverType.String() serverAd.Name = server.Resource @@ -118,30 +122,26 @@ func parseServerAdFromTopology(server server_utils.Server, serverType server_str return serverAd } -// Do a subtraction of excludeDowned set from the includeDowned set to find cache servers -// that are in downtime -// -// The excludeDowned is a list of running OSDF topology servers -// The includeDowned is a list of running and downed OSDF topology servers -func findDownedTopologyCache(excludeDowned, includeDowned []server_utils.Server) (caches []server_utils.Server) { - for _, included := range includeDowned { - found := false - for _, excluded := range excludeDowned { - if included == excluded { - found = true - break - } - } - if !found { - caches = append(caches, included) - } +// Use the topology downtime endpoint to create the list of downed servers. Servers are tracked using their +// resource name, NOT their FQDN. +func updateDowntimeFromTopology(ctx context.Context) error { + dtUrlStr := param.Federation_TopologyDowntimeUrl.GetString() + _, err := url.Parse(dtUrlStr) + if err != nil { + return errors.Wrapf(err, "encountered an invalid URL %s when parsing configured topology downtime URL", dtUrlStr) + } + tr := config.GetTransport() + resp, err := utils.MakeRequest(ctx, tr, dtUrlStr, http.MethodGet, nil, nil) + if err != nil { + return errors.Wrapf(err, "failed to fetch topology downtime from %s", dtUrlStr) } - return -} -// Update filteredServers based on topology downtime -func updateDowntimeFromTopology(excludedNss, includedNss *server_utils.TopologyNamespacesJSON) { - downedCaches := findDownedTopologyCache(excludedNss.Caches, includedNss.Caches) + // Parse the big blurb of XML into a struct. + var downtimeInfo server_structs.TopoDowntimeInfo + err = xml.Unmarshal(resp, &downtimeInfo) + if err != nil { + return errors.Wrap(err, "failed to unmarshal topology downtime XML") + } filteredServersMutex.Lock() defer filteredServersMutex.Unlock() @@ -151,33 +151,44 @@ func updateDowntimeFromTopology(excludedNss, includedNss *server_utils.TopologyN delete(filteredServers, key) } } - for _, dc := range downedCaches { - if sAd := serverAds.Get(dc.Endpoint); sAd == nil { - // The downed cache is not in the director yet - filteredServers[dc.Resource] = topoFiltered - } else { - // If we have the cache in the director, use it's name as the key - filteredServers[sAd.Value().Name] = topoFiltered + + const timeLayout = "Jan 2, 2006 15:04 PM MST" // see https://pkg.go.dev/time#pkg-constants + for _, downtime := range downtimeInfo.CurrentDowntimes.Downtimes { + parsedStartDT, err := time.Parse(timeLayout, downtime.StartTime) + if err != nil { + log.Warningf("Could not put %s into downtime because its start time '%s' could not be parsed: %s", downtime.ResourceName, downtime.StartTime, err) + continue + } + + parsedEndDT, err := time.Parse(timeLayout, downtime.EndTime) + if err != nil { + log.Warningf("Could not put %s into downtime because its end time '%s' could not be parsed: %s", downtime.ResourceName, downtime.EndTime, err) + continue + } + + currentTime := time.Now() + if parsedStartDT.Before(currentTime) && parsedEndDT.After(currentTime) { + filteredServers[downtime.ResourceName] = topoFiltered } } - log.Infof("The following servers are put in downtime: %#v", filteredServers) + + log.Infof("The following servers are currently configured in downtime: %#v", filteredServers) + return nil } // Populate internal cache with origin/cache ads func AdvertiseOSDF(ctx context.Context) error { - namespaces, err := server_utils.GetTopologyJSON(ctx, false) + namespaces, err := server_utils.GetTopologyJSON(ctx) if err != nil { return errors.Wrapf(err, "Failed to get topology JSON") } - // Second call to fetch all servers (including servers in downtime) - includedNss, err := server_utils.GetTopologyJSON(ctx, true) + err = updateDowntimeFromTopology(ctx) if err != nil { - return errors.Wrapf(err, "Failed to get topology JSON with server in downtime included (include_downed)") + // Don't treat this as a fatal error, but log it in a loud way. + log.Errorf("Unable to generate downtime list for servers from topology: %v", err) } - updateDowntimeFromTopology(namespaces, includedNss) - cacheAdMap := make(map[string]*server_structs.Advertisement) // key is serverAd.URL.String() originAdMap := make(map[string]*server_structs.Advertisement) // key is serverAd.URL.String() tGen := server_structs.TokenGen{} diff --git a/director/advertise_test.go b/director/advertise_test.go index 3ec334d92..5a2e05c3a 100644 --- a/director/advertise_test.go +++ b/director/advertise_test.go @@ -19,12 +19,15 @@ package director import ( + "bytes" "context" _ "embed" "net/http" "net/http/httptest" "net/url" "testing" + "text/template" + "time" "github.com/sirupsen/logrus" logrustest "github.com/sirupsen/logrus/hooks/test" @@ -33,7 +36,6 @@ import ( "github.com/stretchr/testify/require" "github.com/pelicanplatform/pelican/server_structs" - "github.com/pelicanplatform/pelican/server_utils" ) var ( @@ -98,7 +100,7 @@ func TestConsolidateDupServerAd(t *testing.T) { func TestParseServerAdFromTopology(t *testing.T) { - server := server_utils.Server{ + server := server_structs.TopoServer{ Endpoint: "http://my-endpoint.com", AuthEndpoint: "https://my-auth-endpoint.com", Resource: "MY_SERVER", @@ -301,149 +303,85 @@ func TestAdvertiseOSDF(t *testing.T) { }) } -func TestFindDownedTopologyCache(t *testing.T) { - mockTopoCacheA := server_utils.Server{AuthEndpoint: "cacheA.org:8443", Endpoint: "cacheA.org:8000", Resource: "CACHE_A"} - mockTopoCacheB := server_utils.Server{AuthEndpoint: "cacheB.org:8443", Endpoint: "cacheB.org:8000", Resource: "CACHE_B"} - mockTopoCacheC := server_utils.Server{AuthEndpoint: "cacheC.org:8443", Endpoint: "cacheC.org:8000", Resource: "CACHE_C"} - mockTopoCacheD := server_utils.Server{AuthEndpoint: "cacheD.org:8443", Endpoint: "cacheD.org:8000", Resource: "CACHE_D"} - t.Run("empty-response", func(t *testing.T) { - get := findDownedTopologyCache( - []server_utils.Server{}, - []server_utils.Server{}, - ) - assert.Empty(t, get) - }) - - t.Run("no-downed-cache", func(t *testing.T) { - get := findDownedTopologyCache( - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, - ) - assert.Empty(t, get) - }) - - t.Run("one-downed-cache", func(t *testing.T) { - get := findDownedTopologyCache( - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC}, - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, - ) - require.Len(t, get, 1) - assert.EqualValues(t, mockTopoCacheD, get[0]) - }) +func mockTopoDowntimeXMLHandler(w http.ResponseWriter, r *http.Request) { + downtimeInfo := server_structs.TopoCurrentDowntimes{ + Downtimes: []server_structs.TopoServerDowntime{ + { + // Current time falls in start-end window. SHould be filtered + ResourceName: "BOISE_INTERNET2_OSDF_CACHE", + ResourceFQDN: "dtn-pas.bois.nrp.internet2.edu", + StartTime: time.Now().Add(-24 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + EndTime: time.Now().Add(24 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + }, + { + // start time is after current time. Should NOT be filtered + ResourceName: "DENVER_INTERNET2_OSDF_CACHE", + ResourceFQDN: "dtn-pas.denv.nrp.internet2.edu", + StartTime: time.Now().Add(24 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + EndTime: time.Now().Add(25 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + }, + { + // end time is before current time. Should NOT be filtered + ResourceName: "HOW_MUCH_CASH_COULD_A_STASHCACHE_STASH", + ResourceFQDN: "stash-cache.cache.osdf.biz", + StartTime: time.Now().Add(-24 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + EndTime: time.Now().Add(-1 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + }, + { + // Invalid time should cause updateDowntimeFromTopology to log an error but not return one + ResourceName: "FOOBAR", + ResourceFQDN: "foo.bar", + StartTime: "The second of January, 2006 03:04 PM MST", + EndTime: time.Now().Add(1 * time.Hour).Format("Jan 2, 2006 03:04 PM MST"), + }, + }, + } - t.Run("two-downed-cache", func(t *testing.T) { - get := findDownedTopologyCache( - []server_utils.Server{mockTopoCacheB, mockTopoCacheC}, - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, - ) - require.Len(t, get, 2) - assert.EqualValues(t, mockTopoCacheA, get[0]) - assert.EqualValues(t, mockTopoCacheD, get[1]) - }) + tmpl, err := template.ParseFiles("resources/mock_topology_downtime_template.xml") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } - t.Run("all-downed-cache", func(t *testing.T) { - get := findDownedTopologyCache( - []server_utils.Server{}, - []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, - ) - assert.EqualValues(t, []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC, mockTopoCacheD}, get) - }) + w.Header().Set("Content-Type", "application/xml") + err = tmpl.Execute(w, downtimeInfo) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } } func TestUpdateDowntimeFromTopology(t *testing.T) { - mockTopoCacheA := server_utils.Server{AuthEndpoint: "cacheA.org:8443", Endpoint: "cacheA.org:8000", Resource: "CACHE_A"} - mockTopoCacheB := server_utils.Server{AuthEndpoint: "cacheB.org:8443", Endpoint: "cacheB.org:8000", Resource: "CACHE_B"} - mockTopoCacheC := server_utils.Server{AuthEndpoint: "cacheC.org:8443", Endpoint: "cacheC.org:8000", Resource: "CACHE_C"} - - t.Run("no-change-with-same-downtime", func(t *testing.T) { - filteredServers = map[string]filterType{} - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{}, - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB}}, - ) - checkResult := func() { - filteredServersMutex.RLock() - defer filteredServersMutex.RUnlock() - assert.Len(t, filteredServers, 2) - require.NotEmpty(t, filteredServers[mockTopoCacheA.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheA.Resource]) - require.NotEmpty(t, filteredServers[mockTopoCacheB.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheB.Resource]) - } - checkResult() - - // second round of updates - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{}, - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB}}, - ) - // Same result - checkResult() + // Create a buffer to capture log output + var logBuffer bytes.Buffer + originalOutput := logrus.StandardLogger().Out + logrus.SetOutput(&logBuffer) + t.Cleanup(func() { + logrus.SetOutput(originalOutput) }) - t.Run("one-server-back-online", func(t *testing.T) { - filteredServers = map[string]filterType{} - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{}, - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB}}, - ) - func() { - filteredServersMutex.RLock() - defer filteredServersMutex.RUnlock() - assert.Len(t, filteredServers, 2) - require.NotEmpty(t, filteredServers[mockTopoCacheA.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheA.Resource]) - require.NotEmpty(t, filteredServers[mockTopoCacheB.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheB.Resource]) - }() + server := httptest.NewServer(http.HandlerFunc(mockTopoDowntimeXMLHandler)) + t.Cleanup(func() { + server.Close() + }) + viper.Set("Federation.TopologyDowntimeUrl", server.URL) - // second round of updates - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA}}, // A is back online - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB}}, - ) - - func() { - filteredServersMutex.RLock() - defer filteredServersMutex.RUnlock() - assert.Len(t, filteredServers, 1) - require.NotEmpty(t, filteredServers[mockTopoCacheB.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheB.Resource]) - }() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(func() { + cancel() }) - t.Run("one-more-server-in-downtime", func(t *testing.T) { - filteredServers = map[string]filterType{} - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{}, - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB}}, - ) - func() { - filteredServersMutex.RLock() - defer filteredServersMutex.RUnlock() - assert.Len(t, filteredServers, 2) - require.NotEmpty(t, filteredServers[mockTopoCacheA.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheA.Resource]) - require.NotEmpty(t, filteredServers[mockTopoCacheB.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheB.Resource]) - }() + err := updateDowntimeFromTopology(ctx) + if err != nil { + t.Fatalf("updateDowntimeFromTopology() error = %v", err) + } - // second round of updates - updateDowntimeFromTopology( - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{}}, - &server_utils.TopologyNamespacesJSON{Caches: []server_utils.Server{mockTopoCacheA, mockTopoCacheB, mockTopoCacheC}}, - ) - - func() { - filteredServersMutex.RLock() - defer filteredServersMutex.RUnlock() - assert.Len(t, filteredServers, 3) - require.NotEmpty(t, filteredServers[mockTopoCacheA.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheA.Resource]) - require.NotEmpty(t, filteredServers[mockTopoCacheB.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheB.Resource]) - require.NotEmpty(t, filteredServers[mockTopoCacheC.Resource]) - assert.Equal(t, topoFiltered, filteredServers[mockTopoCacheC.Resource]) - }() - }) + // There should be a logged warning about the invalid time + logOutput := logBuffer.String() + assert.Contains(t, logOutput, "Could not put FOOBAR into downtime because its start time") + + assert.True(t, filteredServers["BOISE_INTERNET2_OSDF_CACHE"] == topoFiltered) + _, keyExists := filteredServers["DENVER_INTERNET2_OSDF_CACHE"] + assert.False(t, keyExists, "DENVER_INTERNET2_OSDF_CACHE should not be in filteredServers") + _, keyExists = filteredServers["HOW_MUCH_CASH_COULD_A_STASHCACHE_STASH"] + assert.False(t, keyExists, "HOW_MUCH_CASH_COULD_A_STASHCACHE_STASH should not be in filteredServers") } diff --git a/director/resources/mock_topology_downtime_template.xml b/director/resources/mock_topology_downtime_template.xml new file mode 100644 index 000000000..679de818e --- /dev/null +++ b/director/resources/mock_topology_downtime_template.xml @@ -0,0 +1,30 @@ + + +{{- range .Downtimes }} + + 1890864242 + 1405 + + I2BoiseInfrastructure + 1338 + + {{ .ResourceName }} + {{ .ResourceFQDN }} + {{ .StartTime }} + {{ .EndTime }} + UNSCHEDULED + Outage + Aug 19, 2024 16:53 PM UTC + Not Available + + + 156 + XRootD cache server + Internet2 Boise Cache + + + HW issues + +{{- end }} + + diff --git a/docs/parameters.yaml b/docs/parameters.yaml index a4ce75510..41e25cf70 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -348,6 +348,14 @@ osdf_default: https://topology.opensciencegrid.org/osdf/namespaces default: none components: ["director", "registry"] --- +name: Federation.TopologyDowntimeUrl +description: |+ + A URL for determining OSG topology server downtime information. The result of querying this URL is an XML file containing downtime information. +type: url +osdf_default: https://topology.opensciencegrid.org/rgdowntime/xml +default: none +components: ["director"] +--- name: Federation.TopologyReloadInterval description: |+ The frequency, in minutes, that topology should be reloaded. @@ -1366,8 +1374,8 @@ components: ["director"] --- name: Director.FilteredServers description: |+ - A list of server host names to not to redirect client requests to. This is for admins to put a list of - servers in the federation into downtime. + A list of server resource names that the Director should consider in downtime, preventing the Director from issuing redirects to them. + Additional downtimes are aggregated from Topology (when the Director is served in OSDF mode), and the Web UI. type: stringSlice default: none components: ["director"] diff --git a/param/parameters.go b/param/parameters.go index 32360b1ef..7a5b73e94 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -154,6 +154,7 @@ var ( Director_SupportContactEmail = StringParam{"Director.SupportContactEmail"} Director_SupportContactUrl = StringParam{"Director.SupportContactUrl"} Federation_DiscoveryUrl = StringParam{"Federation.DiscoveryUrl"} + Federation_TopologyDowntimeUrl = StringParam{"Federation.TopologyDowntimeUrl"} Federation_TopologyNamespaceUrl = StringParam{"Federation.TopologyNamespaceUrl"} Federation_TopologyUrl = StringParam{"Federation.TopologyUrl"} IssuerKey = StringParam{"IssuerKey"} diff --git a/param/parameters_struct.go b/param/parameters_struct.go index 32eb78faf..f4acfaf61 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -87,6 +87,7 @@ type Config struct { DiscoveryUrl string `mapstructure:"discoveryurl"` JwkUrl string `mapstructure:"jwkurl"` RegistryUrl string `mapstructure:"registryurl"` + TopologyDowntimeUrl string `mapstructure:"topologydowntimeurl"` TopologyNamespaceUrl string `mapstructure:"topologynamespaceurl"` TopologyReloadInterval time.Duration `mapstructure:"topologyreloadinterval"` TopologyUrl string `mapstructure:"topologyurl"` @@ -381,6 +382,7 @@ type configWithType struct { DiscoveryUrl struct { Type string; Value string } JwkUrl struct { Type string; Value string } RegistryUrl struct { Type string; Value string } + TopologyDowntimeUrl struct { Type string; Value string } TopologyNamespaceUrl struct { Type string; Value string } TopologyReloadInterval struct { Type string; Value time.Duration } TopologyUrl struct { Type string; Value string } diff --git a/registry/registry_db.go b/registry/registry_db.go index 2296f23ee..3d8afb9bc 100644 --- a/registry/registry_db.go +++ b/registry/registry_db.go @@ -520,7 +520,7 @@ func PopulateTopology(ctx context.Context) error { } // Next, get the values from topology - namespaces, err := server_utils.GetTopologyJSON(ctx, false) + namespaces, err := server_utils.GetTopologyJSON(ctx) if err != nil { return errors.Wrapf(err, "Failed to get topology JSON") } diff --git a/server_structs/topology.go b/server_structs/topology.go new file mode 100644 index 000000000..bb1d4c9b5 --- /dev/null +++ b/server_structs/topology.go @@ -0,0 +1,84 @@ +package server_structs + +import ( + "encoding/xml" +) + +type ( + TopoServer struct { + AuthEndpoint string `json:"auth_endpoint"` + Endpoint string `json:"endpoint"` + Resource string `json:"resource"` + } + + TopoScitokens struct { + BasePath []string `json:"base_path"` + Issuer string `json:"issuer"` + Restricted []string `json:"restricted_path"` + } + + TopoCredentialGeneration struct { + BasePath string `json:"base_path"` + Issuer string `json:"issuer"` + MaxScopeDepth int `json:"max_scope_depth"` + Strategy string `json:"strategy"` + VaultIssuer string `json:"vault_issuer"` + VaultServer string `json:"vault_server"` + } + + TopoNamespace struct { + Caches []TopoServer `json:"caches"` + Origins []TopoServer `json:"origins"` + CredentialGeneration TopoCredentialGeneration `json:"credential_generation"` + DirlistHost string `json:"dirlisthost"` + Path string `json:"path"` + ReadHTTPS bool `json:"readhttps"` + Scitokens []TopoScitokens `json:"scitokens"` + UseTokenOnRead bool `json:"usetokenonread"` + WritebackHost string `json:"writebackhost"` + } + + TopologyNamespacesJSON struct { + Caches []TopoServer `json:"caches"` + Namespaces []TopoNamespace `json:"namespaces"` + } + + // Structs for encoding downtimes + + TopoResourceGroup struct { + GroupName string `xml:"GroupName"` + GroupID int `xml:"GroupID"` + } + + TopoServices struct { + Service []TopoService `xml:"Service"` + } + + TopoService struct { + ID int `xml:"ID"` + Name string `xml:"Name"` + Description string `xml:"Description"` + } + + TopoDowntimeInfo struct { + XMLName xml.Name `xml:"Downtimes"` + CurrentDowntimes TopoCurrentDowntimes `xml:"CurrentDowntimes"` + } + + TopoCurrentDowntimes struct { + Downtimes []TopoServerDowntime `xml:"Downtime"` + } + + TopoServerDowntime struct { + ID int `xml:"ID"` + ResourceGroup TopoResourceGroup `xml:"ResourceGroup"` + ResourceName string `xml:"ResourceName"` + ResourceFQDN string `xml:"ResourceFQDN"` + StartTime string `xml:"StartTime"` + EndTime string `xml:"EndTime"` + CreatedTime string `xml:"CreatedTime"` + UpdateTime string `xml:"UpdateTime"` + Services TopoServices `xml:"Services"` + Description string `xml:"Description"` + } +) diff --git a/server_utils/server_utils.go b/server_utils/server_utils.go index e13e91f5f..45447d4c9 100644 --- a/server_utils/server_utils.go +++ b/server_utils/server_utils.go @@ -41,50 +41,11 @@ import ( "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/param" -) - -type ( - Server struct { - AuthEndpoint string `json:"auth_endpoint"` - Endpoint string `json:"endpoint"` - Resource string `json:"resource"` - } - - Scitokens struct { - BasePath []string `json:"base_path"` - Issuer string `json:"issuer"` - Restricted []string `json:"restricted_path"` - } - - CredentialGeneration struct { - BasePath string `json:"base_path"` - Issuer string `json:"issuer"` - MaxScopeDepth int `json:"max_scope_depth"` - Strategy string `json:"strategy"` - VaultIssuer string `json:"vault_issuer"` - VaultServer string `json:"vault_server"` - } - - Namespace struct { - Caches []Server `json:"caches"` - Origins []Server `json:"origins"` - CredentialGeneration CredentialGeneration `json:"credential_generation"` - DirlistHost string `json:"dirlisthost"` - Path string `json:"path"` - ReadHTTPS bool `json:"readhttps"` - Scitokens []Scitokens `json:"scitokens"` - UseTokenOnRead bool `json:"usetokenonread"` - WritebackHost string `json:"writebackhost"` - } - - TopologyNamespacesJSON struct { - Caches []Server `json:"caches"` - Namespaces []Namespace `json:"namespaces"` - } + "github.com/pelicanplatform/pelican/server_structs" ) // GetTopologyJSON returns the namespaces and caches from OSDF topology -func GetTopologyJSON(ctx context.Context, includeDowned bool) (*TopologyNamespacesJSON, error) { +func GetTopologyJSON(ctx context.Context) (*server_structs.TopologyNamespacesJSON, error) { topoNamespaceUrl := param.Federation_TopologyNamespaceUrl.GetString() if topoNamespaceUrl == "" { metrics.SetComponentHealthStatus(metrics.DirectorRegistry_Topology, metrics.StatusCritical, "Topology namespaces.json configuration option (`Federation.TopologyNamespaceURL`) not set") @@ -100,9 +61,6 @@ func GetTopologyJSON(ctx context.Context, includeDowned bool) (*TopologyNamespac req.Header.Set("Accept", "application/json") q := req.URL.Query() - if includeDowned { - q.Add("include_downed", "1") - } req.URL.RawQuery = q.Encode() // Use the transport to include timeouts @@ -125,7 +83,7 @@ func GetTopologyJSON(ctx context.Context, includeDowned bool) (*TopologyNamespac return nil, errors.Wrap(err, "Failure when reading OSDF namespace response") } - var namespaces TopologyNamespacesJSON + var namespaces server_structs.TopologyNamespacesJSON if err = json.Unmarshal(respBytes, &namespaces); err != nil { metrics.SetComponentHealthStatus(metrics.DirectorRegistry_Topology, metrics.StatusCritical, fmt.Sprintf("Failure when parsing JSON response from topology URL %v", topoNamespaceUrl)) return nil, errors.Wrapf(err, "Failure when parsing JSON response from topology URL %v", topoNamespaceUrl)