From 3ebc86f87a0a03577db5d03bf0720dbdb8bdfea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Tue, 19 Jan 2021 09:43:17 +0100 Subject: [PATCH] Support site authorization status in Mentix (#1398) --- changelog/unreleased/mentix-site-auth.md | 7 ++ .../config/http/services/mentix/_index.md | 3 + .../http/services/mentix/adminapi/_index.md | 46 ++++++++++++ .../http/services/mentix/cs3api/_index.md | 10 ++- .../http/services/mentix/siteloc/_index.md | 10 ++- .../http/services/mentix/webapi/_index.md | 20 ++++- examples/mentix/mentix.toml | 6 ++ internal/http/services/mentix/mentix.go | 28 +++++-- pkg/mentix/config/config.go | 10 +++ pkg/mentix/config/ids.go | 2 + pkg/mentix/connectors/gocdb.go | 6 ++ pkg/mentix/connectors/localfile.go | 66 +++++++++++++++-- pkg/mentix/exchangers/exporters/cs3api.go | 2 +- pkg/mentix/exchangers/exporters/exporter.go | 19 +++++ pkg/mentix/exchangers/exporters/promsd.go | 9 ++- .../exchangers/exporters/sitelocations.go | 2 +- pkg/mentix/exchangers/exporters/webapi.go | 4 +- pkg/mentix/exchangers/importers/adminapi.go | 60 +++++++++++++++ .../exchangers/importers/adminapi/query.go | 73 +++++++++++++++++++ .../exchangers/importers/reqimporter.go | 15 +++- pkg/mentix/exchangers/importers/webapi.go | 16 ++-- .../exchangers/importers/webapi/query.go | 17 +++-- pkg/mentix/exchangers/reqexchanger.go | 15 +++- pkg/mentix/meshdata/meshdata.go | 24 +++--- pkg/mentix/meshdata/properties.go | 12 +++ pkg/mentix/meshdata/site.go | 11 ++- pkg/sdk/common/net/tus.go | 9 +-- 27 files changed, 441 insertions(+), 61 deletions(-) create mode 100644 changelog/unreleased/mentix-site-auth.md create mode 100644 docs/content/en/docs/config/http/services/mentix/adminapi/_index.md create mode 100755 pkg/mentix/exchangers/importers/adminapi.go create mode 100755 pkg/mentix/exchangers/importers/adminapi/query.go diff --git a/changelog/unreleased/mentix-site-auth.md b/changelog/unreleased/mentix-site-auth.md new file mode 100644 index 0000000000..22372c181a --- /dev/null +++ b/changelog/unreleased/mentix-site-auth.md @@ -0,0 +1,7 @@ +Enhancement: Support site authorization status in Mentix + +This enhancement adds support for a site authorization status to Mentix. This way, sites registered via a web app can now be excluded until authorized manually by an administrator. + +Furthermore, Mentix now sets the scheme for Prometheus targets. This allows us to also support monitoring of sites that do not support the default HTTPS scheme. + +https://github.com/cs3org/reva/pull/1398 diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index 0f696886a2..8e12bc5be7 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -45,6 +45,9 @@ __Supported importers:__ - **webapi** Mentix can import mesh data via an HTTP endpoint using the `webapi` importer. Data can be sent to the configured relative endpoint (see [here](webapi)). +- **adminapi** + Some aspects of Mentix can be administered through an HTTP endpoint using the `adminapi` importer. Queries can be sent to the configured relative endpoint (see [here](adminapi)). + ## Exporters Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint. diff --git a/docs/content/en/docs/config/http/services/mentix/adminapi/_index.md b/docs/content/en/docs/config/http/services/mentix/adminapi/_index.md new file mode 100644 index 0000000000..55f29684eb --- /dev/null +++ b/docs/content/en/docs/config/http/services/mentix/adminapi/_index.md @@ -0,0 +1,46 @@ +--- +title: "adminapi" +linkTitle: "adminapi" +weight: 10 +description: > + Configuration for the AdminAPI of the Mentix service +--- + +{{% pageinfo %}} +The AdminAPI of Mentix is a special importer that can be used to administer certain aspects of Mentix. +{{% /pageinfo %}} + +The AdminAPI importer receives instructions/queries through a `POST` request. + +The importer supports one action that must be passed in the URL: +``` +https://sciencemesh.example.com/mentix/admin/?action= +``` +Currently, the following actions are supported: +- `authorize`: Authorizes or unauthorizes a site + +For all actions, the site data must be sent as JSON data. If the call succeeded, status 200 is returned. + +{{% dir name="endpoint" type="string" default="/admin" %}} +The endpoint where the mesh data can be sent to. +{{< highlight toml >}} +[http.services.mentix.importers.adminapi] +endpoint = "/data" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="is_protected" type="bool" default="false" %}} +Whether the endpoint requires authentication. +{{< highlight toml >}} +[http.services.mentix.importers.adminapi] +is_protected = true +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="enabled_connectors" type="[]string" default="" %}} +A list of all enabled connectors for the importer. Must always be provided. +{{< highlight toml >}} +[http.services.mentix.importers.adminapi] +enabled_connectors = ["localfile"] +{{< /highlight >}} +{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md b/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md index d2707bae1c..167de32c17 100644 --- a/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/cs3api/_index.md @@ -10,7 +10,7 @@ description: > The CS3API exporter exposes Mentix data in a format that is compliant with the CS3API `ProviderInfo` structure via an HTTP endpoint. {{% /pageinfo %}} -{{% dir name="endpoint" type="string" default="/" %}} +{{% dir name="endpoint" type="string" default="/cs3" %}} The endpoint where the mesh data can be queried. {{< highlight toml >}} [http.services.mentix.exporters.cs3api] @@ -18,6 +18,14 @@ endpoint = "/data" {{< /highlight >}} {{% /dir %}} +{{% dir name="is_protected" type="bool" default="false" %}} +Whether the endpoint requires authentication. +{{< highlight toml >}} +[http.services.mentix.exporters.cs3api] +is_protected = true +{{< /highlight >}} +{{% /dir %}} + {{% dir name="enabled_connectors" type="[]string" default="*" %}} A list of all enabled connectors for the exporter. {{< highlight toml >}} diff --git a/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md index d33477dbc2..013d8d3f07 100644 --- a/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/siteloc/_index.md @@ -10,7 +10,7 @@ description: > The Site Locations exporter exposes location information of all sites to be consumed by Grafana via an HTTP endpoint. {{% /pageinfo %}} -{{% dir name="endpoint" type="string" default="/" %}} +{{% dir name="endpoint" type="string" default="/siteloc" %}} The endpoint where the locations data can be queried. {{< highlight toml >}} [http.services.mentix.exporters.siteloc] @@ -18,6 +18,14 @@ endpoint = "/loc" {{< /highlight >}} {{% /dir %}} +{{% dir name="is_protected" type="bool" default="false" %}} +Whether the endpoint requires authentication. +{{< highlight toml >}} +[http.services.mentix.exporters.siteloc] +is_protected = true +{{< /highlight >}} +{{% /dir %}} + {{% dir name="enabled_connectors" type="[]string" default="*" %}} A list of all enabled connectors for the exporter. {{< highlight toml >}} diff --git a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md index 9cf4eaea6c..03e4c028bc 100644 --- a/docs/content/en/docs/config/http/services/mentix/webapi/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/webapi/_index.md @@ -24,7 +24,7 @@ Currently, the following actions are supported: For all actions, the site data must be sent as JSON data. If the call succeeded, status 200 is returned. -{{% dir name="endpoint" type="string" default="/" %}} +{{% dir name="endpoint" type="string" default="/sites" %}} The endpoint where the mesh data can be sent to. {{< highlight toml >}} [http.services.mentix.importers.webapi] @@ -32,6 +32,14 @@ endpoint = "/data" {{< /highlight >}} {{% /dir %}} +{{% dir name="is_protected" type="bool" default="false" %}} +Whether the endpoint requires authentication. +{{< highlight toml >}} +[http.services.mentix.importers.webapi] +is_protected = true +{{< /highlight >}} +{{% /dir %}} + {{% dir name="enabled_connectors" type="[]string" default="" %}} A list of all enabled connectors for the importer. Must always be provided. {{< highlight toml >}} @@ -44,7 +52,7 @@ enabled_connectors = ["localfile"] The WebAPI exporter exposes the _plain_ Mentix data via an HTTP endpoint. -{{% dir name="endpoint" type="string" default="/" %}} +{{% dir name="endpoint" type="string" default="/sites" %}} The endpoint where the mesh data can be queried. {{< highlight toml >}} [http.services.mentix.exporters.webapi] @@ -52,6 +60,14 @@ endpoint = "/data" {{< /highlight >}} {{% /dir %}} +{{% dir name="is_protected" type="bool" default="false" %}} +Whether the endpoint requires authentication. +{{< highlight toml >}} +[http.services.mentix.exporters.webapi] +is_protected = true +{{< /highlight >}} +{{% /dir %}} + {{% dir name="enabled_connectors" type="[]string" default="*" %}} A list of all enabled connectors for the exporter. {{< highlight toml >}} diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index e82c6f8ee1..9baf22bbfc 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -30,6 +30,12 @@ enabled_connectors = ["gocdb"] # For importers, this is obligatory; the connectors will be used as the target for data updates enabled_connectors = ["localfile"] +# Enable the AdminAPI importer +[http.services.mentix.importers.adminapi] +enabled_connectors = ["localfile"] +# Should never allow access w/o prior authorization +is_protected = true + # Configure the Prometheus Service Discovery: [http.services.mentix.exporters.promsd] # The following files must be made available to Prometheus. diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 29541b3253..f07070d31d 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -27,6 +27,7 @@ import ( "github.com/cs3org/reva/pkg/mentix" "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/rhttp/global" ) @@ -63,13 +64,20 @@ func (s *svc) Unprotected() []string { importers := s.mntx.GetRequestImporters() exporters := s.mntx.GetRequestExporters() - endpoints := make([]string, len(importers)+len(exporters)) - for idx, importer := range importers { - endpoints[idx] = importer.Endpoint() - } - for idx, exporter := range exporters { - endpoints[idx] = exporter.Endpoint() + getEndpoints := func(exchangers []exchangers.RequestExchanger) []string { + endpoints := make([]string, 0, len(exchangers)) + for _, exchanger := range exchangers { + if !exchanger.IsProtectedEndpoint() { + endpoints = append(endpoints, exchanger.Endpoint()) + } + } + return endpoints } + + endpoints := make([]string, 0, len(importers)+len(exporters)) + endpoints = append(endpoints, getEndpoints(importers)...) + endpoints = append(endpoints, getEndpoints(exporters)...) + return endpoints } @@ -130,7 +138,11 @@ func applyDefaultConfig(conf *config.Configuration) { // Importers if conf.Importers.WebAPI.Endpoint == "" { - conf.Importers.WebAPI.Endpoint = "/" + conf.Importers.WebAPI.Endpoint = "/sites" + } + + if conf.Importers.AdminAPI.Endpoint == "" { + conf.Importers.AdminAPI.Endpoint = "/admin" } // Exporters @@ -141,7 +153,7 @@ func applyDefaultConfig(conf *config.Configuration) { } if conf.Exporters.WebAPI.Endpoint == "" { - conf.Exporters.WebAPI.Endpoint = "/" + conf.Exporters.WebAPI.Endpoint = "/sites" } addDefaultConnector(&conf.Exporters.WebAPI.EnabledConnectors) diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index d37d64e383..b84ded160f 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -38,23 +38,33 @@ type Configuration struct { Importers struct { WebAPI struct { Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"webapi"` + + AdminAPI struct { + Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` + EnabledConnectors []string `mapstructure:"enabled_connectors"` + } `mapstructure:"adminapi"` } `mapstructure:"importers"` Exporters struct { WebAPI struct { Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"webapi"` CS3API struct { Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"cs3api"` SiteLocations struct { Endpoint string `mapstructure:"endpoint"` + IsProtected bool `mapstructure:"is_protected"` EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"siteloc"` diff --git a/pkg/mentix/config/ids.go b/pkg/mentix/config/ids.go index 04ff545ec3..9fbb020693 100644 --- a/pkg/mentix/config/ids.go +++ b/pkg/mentix/config/ids.go @@ -28,6 +28,8 @@ const ( const ( // ImporterIDWebAPI is the identifier for the WebAPI importer. ImporterIDWebAPI = "webapi" + // ImporterIDAdminAPI is the identifier for the AdminAPI importer. + ImporterIDAdminAPI = "adminapi" ) const ( diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 7551775894..75742f27d3 100755 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -75,6 +75,7 @@ func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error) } } + meshData.InferMissingData() return meshData, nil } @@ -127,6 +128,11 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error { for _, site := range sites.Sites { properties := connector.extensionsToMap(&site.Extensions) + // Sites coming from the GOCDB are always authorized by default + if value := meshdata.GetPropertyValue(properties, meshdata.PropertyAuthorized, ""); len(value) == 0 { + meshdata.SetPropertyValue(&properties, meshdata.PropertyAuthorized, "true") + } + // See if an organization has been defined using properties; otherwise, use the official name organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName) diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go index b5b15eda9a..cbe3da9605 100755 --- a/pkg/mentix/connectors/localfile.go +++ b/pkg/mentix/connectors/localfile.go @@ -78,6 +78,7 @@ func (connector *LocalFileConnector) RetrieveMeshData() (*meshdata.MeshData, err // Update the site types, as these are not part of the JSON data connector.setSiteTypes(meshData) + meshData.InferMissingData() return meshData, nil } @@ -89,12 +90,23 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa meshData = &meshdata.MeshData{} } - if (updatedData.Flags & meshdata.FlagObsolete) == meshdata.FlagObsolete { - // Remove data by unmerging - meshData.Unmerge(updatedData) - } else { - // Add/update data by merging - meshData.Merge(updatedData) + err = nil + switch updatedData.Status { + case meshdata.StatusDefault: + err = connector.mergeData(meshData, updatedData) + + case meshdata.StatusObsolete: + err = connector.unmergeData(meshData, updatedData) + + case meshdata.StatusAuthorize: + err = connector.authorizeData(meshData, updatedData, true) + + case meshdata.StatusUnauthorize: + err = connector.authorizeData(meshData, updatedData, false) + } + + if err != nil { + return err } // Write the updated sites back to the file @@ -106,6 +118,48 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa return nil } +func (connector *LocalFileConnector) mergeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData) error { + // Store the previous authorization status for already existing sites + siteAuthorizationStatus := make(map[string]string) + for _, site := range meshData.Sites { + siteAuthorizationStatus[site.ID] = meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false") + } + + // Add/update data by merging + meshData.Merge(updatedData) + + // Restore the authorization status for all sites + for siteID, status := range siteAuthorizationStatus { + if site := meshData.FindSite(siteID); site != nil { + meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, status) + } + } + return nil +} + +func (connector *LocalFileConnector) unmergeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData) error { + // Remove data by unmerging + meshData.Unmerge(updatedData) + return nil +} + +func (connector *LocalFileConnector) authorizeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData, authorize bool) error { + for _, placeholderSite := range updatedData.Sites { + // The site ID is stored in the updated site's name + if site := meshData.FindSite(placeholderSite.Name); site != nil { + if authorize { + meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "true") + } else { + meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "false") + } + } else { + return fmt.Errorf("no site with id '%v' found", placeholderSite.Name) + } + } + + return nil +} + func (connector *LocalFileConnector) setSiteTypes(meshData *meshdata.MeshData) { for _, site := range meshData.Sites { site.Type = meshdata.SiteTypeCommunity // Sites coming from a local file are always community sites diff --git a/pkg/mentix/exchangers/exporters/cs3api.go b/pkg/mentix/exchangers/exporters/cs3api.go index 1ed15eb7e2..3102c0ce88 100755 --- a/pkg/mentix/exchangers/exporters/cs3api.go +++ b/pkg/mentix/exchangers/exporters/cs3api.go @@ -37,7 +37,7 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo } // Store CS3API specifics - exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint) + exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint, conf.Exporters.CS3API.IsProtected) exporter.SetEnabledConnectors(conf.Exporters.CS3API.EnabledConnectors) exporter.defaultActionHandler = cs3api.HandleDefaultQuery diff --git a/pkg/mentix/exchangers/exporters/exporter.go b/pkg/mentix/exchangers/exporters/exporter.go index 51cd604b40..605ca083f2 100755 --- a/pkg/mentix/exchangers/exporters/exporter.go +++ b/pkg/mentix/exchangers/exporters/exporter.go @@ -20,6 +20,7 @@ package exporters import ( "fmt" + "strings" "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/mentix/meshdata" @@ -41,6 +42,8 @@ type BaseExporter struct { exchangers.BaseExchanger meshData *meshdata.MeshData + + allowUnauthorizedSites bool } // Update is called whenever the mesh data set has changed to reflect these changes. @@ -65,6 +68,11 @@ func (exporter *BaseExporter) storeMeshDataSet(meshDataSet meshdata.Map) error { if meshDataCloned == nil { return fmt.Errorf("unable to clone the mesh data") } + + if !exporter.allowUnauthorizedSites { + exporter.removeUnauthorizedSites(meshDataCloned) + } + meshDataSetCloned[connectorID] = meshDataCloned } exporter.SetMeshData(meshdata.MergeMeshDataMap(meshDataSetCloned)) @@ -84,3 +92,14 @@ func (exporter *BaseExporter) SetMeshData(meshData *meshdata.MeshData) { exporter.meshData = meshData } + +func (exporter *BaseExporter) removeUnauthorizedSites(meshData *meshdata.MeshData) { + cleanedSites := make([]*meshdata.Site, 0, len(meshData.Sites)) + for _, site := range meshData.Sites { + // Only keep authorized sites + if value := meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false"); strings.EqualFold(value, "true") { + cleanedSites = append(cleanedSites, site) + } + } + meshData.Sites = cleanedSites +} diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index bbdf717c23..40d8356244 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net/url" "os" "path/filepath" "strings" @@ -49,6 +50,12 @@ type PrometheusSDExporter struct { func createMetricsSDScrapeConfig(site *meshdata.Site, host string, endpoint *meshdata.ServiceEndpoint) *prometheus.ScrapeConfig { labels := getScrapeTargetLabels(site, host, endpoint) + // Support both HTTP and HTTPS endpoints by setting the scheme label accordingly + if len(endpoint.URL) > 0 { + if url, err := url.Parse(endpoint.URL); err == nil && (url.Scheme == "http" || url.Scheme == "https") { + labels["__scheme__"] = url.Scheme + } + } // If a metrics path was specified as a property, use that one by setting the corresponding label if metricsPath := meshdata.GetPropertyValue(endpoint.Properties, meshdata.PropertyMetricsPath, ""); len(metricsPath) > 0 { labels["__metrics_path__"] = metricsPath @@ -89,7 +96,7 @@ func getScrapeTargetLabels(site *meshdata.Site, host string, endpoint *meshdata. labels := map[string]string{ "__meta_mentix_site": site.Name, "__meta_mentix_site_type": meshdata.GetSiteTypeName(site.Type), - "__meta_mentix_site_id": site.GetID(), + "__meta_mentix_site_id": site.ID, "__meta_mentix_host": host, "__meta_mentix_country": site.CountryCode, "__meta_mentix_service_type": endpoint.Type.Name, diff --git a/pkg/mentix/exchangers/exporters/sitelocations.go b/pkg/mentix/exchangers/exporters/sitelocations.go index f00b8d625a..ce6034f89b 100755 --- a/pkg/mentix/exchangers/exporters/sitelocations.go +++ b/pkg/mentix/exchangers/exporters/sitelocations.go @@ -37,7 +37,7 @@ func (exporter *SiteLocationsExporter) Activate(conf *config.Configuration, log } // Store SiteLocations specifics - exporter.SetEndpoint(conf.Exporters.SiteLocations.Endpoint) + exporter.SetEndpoint(conf.Exporters.SiteLocations.Endpoint, conf.Exporters.SiteLocations.IsProtected) exporter.SetEnabledConnectors(conf.Exporters.SiteLocations.EnabledConnectors) exporter.defaultActionHandler = siteloc.HandleDefaultQuery diff --git a/pkg/mentix/exchangers/exporters/webapi.go b/pkg/mentix/exchangers/exporters/webapi.go index dee73a4f16..f784c3df0d 100755 --- a/pkg/mentix/exchangers/exporters/webapi.go +++ b/pkg/mentix/exchangers/exporters/webapi.go @@ -37,9 +37,11 @@ func (exporter *WebAPIExporter) Activate(conf *config.Configuration, log *zerolo } // Store WebAPI specifics - exporter.SetEndpoint(conf.Exporters.WebAPI.Endpoint) + exporter.SetEndpoint(conf.Exporters.WebAPI.Endpoint, conf.Exporters.WebAPI.IsProtected) exporter.SetEnabledConnectors(conf.Exporters.WebAPI.EnabledConnectors) + exporter.allowUnauthorizedSites = true + exporter.defaultActionHandler = webapi.HandleDefaultQuery return nil diff --git a/pkg/mentix/exchangers/importers/adminapi.go b/pkg/mentix/exchangers/importers/adminapi.go new file mode 100755 index 0000000000..53ee50748b --- /dev/null +++ b/pkg/mentix/exchangers/importers/adminapi.go @@ -0,0 +1,60 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package importers + +import ( + "github.com/rs/zerolog" + + "github.com/cs3org/reva/pkg/mentix/config" + "github.com/cs3org/reva/pkg/mentix/exchangers/importers/adminapi" +) + +// AdminAPIImporter implements the administrative API importer. +type AdminAPIImporter struct { + BaseRequestImporter +} + +// Activate activates the importer. +func (importer *AdminAPIImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := importer.BaseRequestImporter.Activate(conf, log); err != nil { + return err + } + + // Store AdminAPI specifics + importer.SetEndpoint(conf.Importers.AdminAPI.Endpoint, conf.Importers.AdminAPI.IsProtected) + importer.SetEnabledConnectors(conf.Importers.AdminAPI.EnabledConnectors) + + importer.authorizeSiteActionHandler = adminapi.HandleAuthorizeSiteQuery + + return nil +} + +// GetID returns the ID of the importer. +func (importer *AdminAPIImporter) GetID() string { + return config.ImporterIDAdminAPI +} + +// GetName returns the display name of the importer. +func (importer *AdminAPIImporter) GetName() string { + return "AdminAPI" +} + +func init() { + registerImporter(&AdminAPIImporter{}) +} diff --git a/pkg/mentix/exchangers/importers/adminapi/query.go b/pkg/mentix/exchangers/importers/adminapi/query.go new file mode 100755 index 0000000000..36c2265849 --- /dev/null +++ b/pkg/mentix/exchangers/importers/adminapi/query.go @@ -0,0 +1,73 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package adminapi + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/cs3org/reva/pkg/mentix/meshdata" + "github.com/cs3org/reva/pkg/mentix/network" +) + +func decodeAdminQueryData(data []byte) (*meshdata.MeshData, error) { + jsonData := make(map[string]interface{}) + if err := json.Unmarshal(data, &jsonData); err != nil { + return nil, err + } + + if value, ok := jsonData["id"]; ok { + if id, ok := value.(string); ok { + site := &meshdata.Site{} + site.Name = id // Store the provided site ID in the site's name + + meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} + return meshData, nil + } + + return nil, fmt.Errorf("site id invalid") + } + + return nil, fmt.Errorf("site id missing") +} + +func handleAdminQuery(data []byte, params url.Values, status int, msg string) (meshdata.Vector, int, []byte, error) { + meshData, err := decodeAdminQueryData(data) + if err != nil { + return nil, http.StatusBadRequest, network.CreateResponse("INVALID_DATA", network.ResponseParams{"error": err.Error()}), nil + } + meshData.Status = status + return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": meshData.Sites[0].Name}), nil +} + +// HandleAuthorizeSiteQuery sets the authorization status of a site. +func HandleAuthorizeSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { + status := params.Get("status") + + if strings.EqualFold(status, "true") { + return handleAdminQuery(data, params, meshdata.StatusAuthorize, "SITE_AUTHORIZED") + } else if strings.EqualFold(status, "false") { + return handleAdminQuery(data, params, meshdata.StatusUnauthorize, "SITE_UNAUTHORIZED") + } + + return nil, http.StatusBadRequest, network.CreateResponse("INVALID_QUERY", network.ResponseParams{}), nil +} diff --git a/pkg/mentix/exchangers/importers/reqimporter.go b/pkg/mentix/exchangers/importers/reqimporter.go index 2a6a84eeae..673e4ff161 100644 --- a/pkg/mentix/exchangers/importers/reqimporter.go +++ b/pkg/mentix/exchangers/importers/reqimporter.go @@ -32,6 +32,7 @@ import ( const ( queryActionRegisterSite = "register" queryActionUnregisterSite = "unregister" + queryActionAuthorizeSite = "authorize" ) type queryCallback func([]byte, url.Values) (meshdata.Vector, int, []byte, error) @@ -43,14 +44,17 @@ type BaseRequestImporter struct { registerSiteActionHandler queryCallback unregisterSiteActionHandler queryCallback + authorizeSiteActionHandler queryCallback } // HandleRequest handles the actual HTTP request. func (importer *BaseRequestImporter) HandleRequest(resp http.ResponseWriter, req *http.Request) { body, _ := ioutil.ReadAll(req.Body) - meshData, status, respData, err := importer.handleQuery(body, req.URL.Query()) + meshData, status, respData, err := importer.handleQuery(body, req.URL.Path, req.URL.Query()) if err == nil { - importer.mergeImportedMeshData(meshData) + if len(meshData) > 0 { + importer.mergeImportedMeshData(meshData) + } } else { respData = []byte(err.Error()) } @@ -71,7 +75,7 @@ func (importer *BaseRequestImporter) mergeImportedMeshData(meshData meshdata.Vec } } -func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { +func (importer *BaseRequestImporter) handleQuery(data []byte, path string, params url.Values) (meshdata.Vector, int, []byte, error) { action := params.Get("action") switch strings.ToLower(action) { case queryActionRegisterSite: @@ -84,6 +88,11 @@ func (importer *BaseRequestImporter) handleQuery(data []byte, params url.Values) return importer.unregisterSiteActionHandler(data, params) } + case queryActionAuthorizeSite: + if importer.authorizeSiteActionHandler != nil { + return importer.authorizeSiteActionHandler(data, params) + } + default: return nil, http.StatusNotImplemented, []byte{}, fmt.Errorf("unknown action '%v'", action) } diff --git a/pkg/mentix/exchangers/importers/webapi.go b/pkg/mentix/exchangers/importers/webapi.go index d1214a0fd9..508ff62643 100755 --- a/pkg/mentix/exchangers/importers/webapi.go +++ b/pkg/mentix/exchangers/importers/webapi.go @@ -31,28 +31,28 @@ type WebAPIImporter struct { } // Activate activates the importer. -func (exporter *WebAPIImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if err := exporter.BaseRequestImporter.Activate(conf, log); err != nil { +func (importer *WebAPIImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { + if err := importer.BaseRequestImporter.Activate(conf, log); err != nil { return err } // Store WebAPI specifics - exporter.SetEndpoint(conf.Importers.WebAPI.Endpoint) - exporter.SetEnabledConnectors(conf.Importers.WebAPI.EnabledConnectors) + importer.SetEndpoint(conf.Importers.WebAPI.Endpoint, conf.Importers.WebAPI.IsProtected) + importer.SetEnabledConnectors(conf.Importers.WebAPI.EnabledConnectors) - exporter.registerSiteActionHandler = webapi.HandleRegisterSiteQuery - exporter.unregisterSiteActionHandler = webapi.HandleUnregisterSiteQuery + importer.registerSiteActionHandler = webapi.HandleRegisterSiteQuery + importer.unregisterSiteActionHandler = webapi.HandleUnregisterSiteQuery return nil } // GetID returns the ID of the importer. -func (exporter *WebAPIImporter) GetID() string { +func (importer *WebAPIImporter) GetID() string { return config.ImporterIDWebAPI } // GetName returns the display name of the importer. -func (exporter *WebAPIImporter) GetName() string { +func (importer *WebAPIImporter) GetName() string { return "WebAPI" } diff --git a/pkg/mentix/exchangers/importers/webapi/query.go b/pkg/mentix/exchangers/importers/webapi/query.go index aceb4d65b2..868d0c8cc3 100755 --- a/pkg/mentix/exchangers/importers/webapi/query.go +++ b/pkg/mentix/exchangers/importers/webapi/query.go @@ -34,29 +34,36 @@ func decodeQueryData(data []byte) (*meshdata.MeshData, error) { return nil, err } + // Imported sites will be assigned an ID automatically + site.ID = "" + + // Set sites imported through the WebAPI to 'unauthorized' by default + meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "false") + meshData := &meshdata.MeshData{Sites: []*meshdata.Site{site}} if err := meshData.Verify(); err != nil { return nil, fmt.Errorf("verifying the imported mesh data failed: %v", err) } + meshData.InferMissingData() return meshData, nil } -func handleQuery(data []byte, params url.Values, flags int32, msg string) (meshdata.Vector, int, []byte, error) { +func handleQuery(data []byte, params url.Values, status int, msg string) (meshdata.Vector, int, []byte, error) { meshData, err := decodeQueryData(data) if err != nil { return nil, http.StatusBadRequest, network.CreateResponse("INVALID_DATA", network.ResponseParams{"error": err.Error()}), nil } - meshData.Flags = flags - return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": meshData.Sites[0].GetID()}), nil + meshData.Status = status + return meshdata.Vector{meshData}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": meshData.Sites[0].ID}), nil } // HandleRegisterSiteQuery registers a site. func HandleRegisterSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { - return handleQuery(data, params, meshdata.FlagsNone, "SITE_REGISTERED") + return handleQuery(data, params, meshdata.StatusDefault, "SITE_REGISTERED") } // HandleUnregisterSiteQuery unregisters a site. func HandleUnregisterSiteQuery(data []byte, params url.Values) (meshdata.Vector, int, []byte, error) { - return handleQuery(data, params, meshdata.FlagObsolete, "SITE_UNREGISTERED") + return handleQuery(data, params, meshdata.StatusObsolete, "SITE_UNREGISTERED") } diff --git a/pkg/mentix/exchangers/reqexchanger.go b/pkg/mentix/exchangers/reqexchanger.go index 427d99fc05..c4c9e705af 100644 --- a/pkg/mentix/exchangers/reqexchanger.go +++ b/pkg/mentix/exchangers/reqexchanger.go @@ -27,6 +27,8 @@ import ( type RequestExchanger interface { // Endpoint returns the (relative) endpoint of the exchanger. Endpoint() string + // IsProtectedEndpoint returns true if the endpoint can only be accessed with authorization. + IsProtectedEndpoint() bool // WantsRequest returns whether the exchanger wants to handle the incoming request. WantsRequest(r *http.Request) bool // HandleRequest handles the actual HTTP request. @@ -37,7 +39,8 @@ type RequestExchanger interface { type BaseRequestExchanger struct { RequestExchanger - endpoint string + endpoint string + isProtectedEndpoint bool } // Endpoint returns the (relative) endpoint of the exchanger. @@ -50,12 +53,18 @@ func (exchanger *BaseRequestExchanger) Endpoint() string { return strings.TrimSpace(endpoint) } +// IsProtectedEndpoint returns true if the endpoint can only be accessed with authorization. +func (exchanger *BaseRequestExchanger) IsProtectedEndpoint() bool { + return exchanger.isProtectedEndpoint +} + // SetEndpoint sets the (relative) endpoint of the exchanger. -func (exchanger *BaseRequestExchanger) SetEndpoint(endpoint string) { +func (exchanger *BaseRequestExchanger) SetEndpoint(endpoint string, isProtected bool) { exchanger.endpoint = endpoint + exchanger.isProtectedEndpoint = isProtected } -// WantsRequest returns whether the exporter wants to handle the incoming request. +// WantsRequest returns whether the exchanger wants to handle the incoming request. func (exchanger *BaseRequestExchanger) WantsRequest(r *http.Request) bool { return r.URL.Path == exchanger.Endpoint() } diff --git a/pkg/mentix/meshdata/meshdata.go b/pkg/mentix/meshdata/meshdata.go index 23ecfb5185..9ab0ecdf4a 100644 --- a/pkg/mentix/meshdata/meshdata.go +++ b/pkg/mentix/meshdata/meshdata.go @@ -25,11 +25,15 @@ import ( ) const ( - // FlagsNone resets all mesh data flags. - FlagsNone = 0 - - // FlagObsolete flags the mesh data for removal. - FlagObsolete = 0x0001 + // StatusDefault signals that this is just regular data. + StatusDefault = iota + + // StatusObsolete flags the mesh data for removal. + StatusObsolete + // StatusAuthorize flags the mesh data for authorization. + StatusAuthorize + // StatusUnauthorize flags the mesh data for unauthorization. + StatusUnauthorize ) // MeshData holds the entire mesh data managed by Mentix. @@ -37,7 +41,7 @@ type MeshData struct { Sites []*Site ServiceTypes []*ServiceType - Flags int32 `json:"-"` + Status int `json:"-"` } // Clear removes all saved data, leaving an empty mesh. @@ -45,12 +49,12 @@ func (meshData *MeshData) Clear() { meshData.Sites = nil meshData.ServiceTypes = nil - meshData.Flags = FlagsNone + meshData.Status = StatusDefault } // AddSite adds a new site; if a site with the same ID already exists, the existing one is overwritten. func (meshData *MeshData) AddSite(site *Site) { - if siteExisting := meshData.FindSite(site.GetID()); siteExisting != nil { + if siteExisting := meshData.FindSite(site.ID); siteExisting != nil { *siteExisting = *site } else { meshData.Sites = append(meshData.Sites, site) @@ -75,7 +79,7 @@ func (meshData *MeshData) RemoveSite(id string) { // FindSite searches for a site with the given ID. func (meshData *MeshData) FindSite(id string) *Site { for _, site := range meshData.Sites { - if strings.EqualFold(site.GetID(), id) { + if strings.EqualFold(site.ID, id) { return site } } @@ -130,7 +134,7 @@ func (meshData *MeshData) Merge(inData *MeshData) { // Unmerge removes data from another MeshData instance from this one. func (meshData *MeshData) Unmerge(inData *MeshData) { for _, site := range inData.Sites { - meshData.RemoveSite(site.GetID()) + meshData.RemoveSite(site.ID) } for _, serviceType := range inData.ServiceTypes { diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index 524be3c2b0..0374ec2e2f 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -21,6 +21,8 @@ package meshdata import "strings" const ( + // PropertyAuthorized identifies the authorization status property. + PropertyAuthorized = "authorized" // PropertyOrganization identifies the organization property. PropertyOrganization = "organization" // PropertyMetricsPath identifies the metrics path property. @@ -43,3 +45,13 @@ func GetPropertyValue(props map[string]string, id string, defValue string) strin return defValue } + +// SetPropertyValue sets a property value. +func SetPropertyValue(props *map[string]string, id string, value string) { + // If the provided properties map is nil, create an empty one + if *props == nil { + *props = make(map[string]string) + } + + (*props)[strings.ToUpper(id)] = value +} diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index a21432f1fd..ce41174ad5 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -40,6 +40,7 @@ type SiteType int type Site struct { Type SiteType `json:"-"` + ID string Name string FullName string Organization string @@ -122,15 +123,19 @@ func (site *Site) InferMissingData() { } } + // Automatically assign an ID to this site if it is missing + if len(site.ID) == 0 { + site.generateID() + } + // Infer missing for services for _, service := range site.Services { service.InferMissingData() } } -// GetID generates a unique ID for the site; the following fields are used for this: // Name, Domain -func (site *Site) GetID() string { +func (site *Site) generateID() { host := site.Domain if site.Homepage != "" { if hostURL, err := url.Parse(site.Homepage); err == nil { @@ -138,7 +143,7 @@ func (site *Site) GetID() string { } } - return fmt.Sprintf("%s::[%s]", host, site.Name) + site.ID = fmt.Sprintf("%s::[%s]", host, site.Name) } // GetSiteTypeName returns the readable name of the given site type. diff --git a/pkg/sdk/common/net/tus.go b/pkg/sdk/common/net/tus.go index 794e5cbde2..bf4a73df3c 100644 --- a/pkg/sdk/common/net/tus.go +++ b/pkg/sdk/common/net/tus.go @@ -52,13 +52,8 @@ func (client *TUSClient) initClient(endpoint string, accessToken string, transpo } client.config.Store = memStore - if accessToken != "" { - client.config.Header.Add(AccessTokenName, accessToken) - } - - if transportToken != "" { - client.config.Header.Add(TransportTokenName, transportToken) - } + client.config.Header.Add(AccessTokenName, accessToken) + client.config.Header.Add(TransportTokenName, transportToken) // Create the TUS client tusClient, err := tus.NewClient(endpoint, client.config)