Support site authorization status in Mentix

changelog/unreleased/mentix-site-auth.md

@@ -0,0 +1,7 @@
+Enhancement: Support site authorization status in Mentix
+
+This enhancement adds support for a site authorization status to Mentix. This way, sites registered via a web app can now be excluded until authorized manually by an administrator. 
+
+Furthermore, Mentix now sets the scheme for Prometheus targets. This allows us to also support monitoring of sites that do not support the default HTTPS scheme.
+
+https://github.com/cs3org/reva/pull/1398

docs/content/en/docs/config/http/services/mentix/_index.md

@@ -45,6 +45,9 @@ __Supported importers:__
 - **webapi**
 Mentix can import mesh data via an HTTP endpoint using the `webapi` importer. Data can be sent to the configured relative endpoint (see [here](webapi)).
 
+- **adminapi**
+  Some aspects of Mentix can be administered through an HTTP endpoint using the `adminapi` importer. Queries can be sent to the configured relative endpoint (see [here](adminapi)).
+
 ## Exporters
 Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint.
 

docs/content/en/docs/config/http/services/mentix/adminapi/_index.md

@@ -0,0 +1,46 @@
+---
+title: "adminapi"
+linkTitle: "adminapi"
+weight: 10
+description: >
+    Configuration for the AdminAPI of the Mentix service
+---
+
+{{% pageinfo %}}
+The AdminAPI of Mentix is a special importer that can be used to administer certain aspects of Mentix.
+{{% /pageinfo %}}
+
+The AdminAPI importer receives instructions/queries through a `POST` request.
+
+The importer supports one action that must be passed in the URL:
+```
+https://sciencemesh.example.com/mentix/admin/?action=<value>
+```
+Currently, the following actions are supported:
+- `authorize`: Authorizes or unauthorizes a site
+
+For all actions, the site data must be sent as JSON data. If the call succeeded, status 200 is returned.
+
+{{% dir name="endpoint" type="string" default="/admin" %}}
+The endpoint where the mesh data can be sent to.
+{{< highlight toml >}}
+[http.services.mentix.importers.adminapi]
+endpoint = "/data"
+{{< /highlight >}}
+{{% /dir %}}
+
+{{% dir name="is_protected" type="bool" default="false" %}}
+Whether the endpoint requires authentication.
+{{< highlight toml >}}
+[http.services.mentix.importers.adminapi]
+is_protected = true
+{{< /highlight >}}
+{{% /dir %}}
+
+{{% dir name="enabled_connectors" type="[]string" default="" %}}
+A list of all enabled connectors for the importer. Must always be provided.
+{{< highlight toml >}}
+[http.services.mentix.importers.adminapi]
+enabled_connectors = ["localfile"]
+{{< /highlight >}}
+{{% /dir %}}

docs/content/en/docs/config/http/services/mentix/cs3api/_index.md

@@ -10,14 +10,22 @@ description: >
 The CS3API exporter exposes Mentix data in a format that is compliant with the CS3API `ProviderInfo` structure via an HTTP endpoint.
 {{% /pageinfo %}}
 
-{{% dir name="endpoint" type="string" default="/" %}}
+{{% dir name="endpoint" type="string" default="/cs3" %}}
 The endpoint where the mesh data can be queried.
 {{< highlight toml >}}
 [http.services.mentix.exporters.cs3api]
 endpoint = "/data"
 {{< /highlight >}}
 {{% /dir %}}
 
+{{% dir name="is_protected" type="bool" default="false" %}}
+Whether the endpoint requires authentication.
+{{< highlight toml >}}
+[http.services.mentix.exporters.cs3api]
+is_protected = true
+{{< /highlight >}}
+{{% /dir %}}
+
 {{% dir name="enabled_connectors" type="[]string" default="*" %}}
 A list of all enabled connectors for the exporter.
 {{< highlight toml >}}

docs/content/en/docs/config/http/services/mentix/siteloc/_index.md

@@ -10,14 +10,22 @@ description: >
 The Site Locations exporter exposes location information of all sites to be consumed by Grafana via an HTTP endpoint.
 {{% /pageinfo %}}
 
-{{% dir name="endpoint" type="string" default="/" %}}
+{{% dir name="endpoint" type="string" default="/siteloc" %}}
 The endpoint where the locations data can be queried.
 {{< highlight toml >}}
 [http.services.mentix.exporters.siteloc]
 endpoint = "/loc"
 {{< /highlight >}}
 {{% /dir %}}
 
+{{% dir name="is_protected" type="bool" default="false" %}}
+Whether the endpoint requires authentication.
+{{< highlight toml >}}
+[http.services.mentix.exporters.siteloc]
+is_protected = true
+{{< /highlight >}}
+{{% /dir %}}
+
 {{% dir name="enabled_connectors" type="[]string" default="*" %}}
 A list of all enabled connectors for the exporter.
 {{< highlight toml >}}

docs/content/en/docs/config/http/services/mentix/webapi/_index.md

@@ -24,14 +24,22 @@ Currently, the following actions are supported:
 
 For all actions, the site data must be sent as JSON data. If the call succeeded, status 200 is returned.
 
-{{% dir name="endpoint" type="string" default="/" %}}
+{{% dir name="endpoint" type="string" default="/sites" %}}
 The endpoint where the mesh data can be sent to.
 {{< highlight toml >}}
 [http.services.mentix.importers.webapi]
 endpoint = "/data"
 {{< /highlight >}}
 {{% /dir %}}
 
+{{% dir name="is_protected" type="bool" default="false" %}}
+Whether the endpoint requires authentication.
+{{< highlight toml >}}
+[http.services.mentix.importers.webapi]
+is_protected = true
+{{< /highlight >}}
+{{% /dir %}}
+
 {{% dir name="enabled_connectors" type="[]string" default="" %}}
 A list of all enabled connectors for the importer. Must always be provided.
 {{< highlight toml >}}
@@ -44,14 +52,22 @@ enabled_connectors = ["localfile"]
 
 The WebAPI exporter exposes the _plain_ Mentix data via an HTTP endpoint.
 
-{{% dir name="endpoint" type="string" default="/" %}}
+{{% dir name="endpoint" type="string" default="/sites" %}}
 The endpoint where the mesh data can be queried.
 {{< highlight toml >}}
 [http.services.mentix.exporters.webapi]
 endpoint = "/data"
 {{< /highlight >}}
 {{% /dir %}}
 
+{{% dir name="is_protected" type="bool" default="false" %}}
+Whether the endpoint requires authentication.
+{{< highlight toml >}}
+[http.services.mentix.exporters.webapi]
+is_protected = true
+{{< /highlight >}}
+{{% /dir %}}
+
 {{% dir name="enabled_connectors" type="[]string" default="*" %}}
 A list of all enabled connectors for the exporter.
 {{< highlight toml >}}

examples/mentix/mentix.toml

@@ -30,6 +30,12 @@ enabled_connectors = ["gocdb"]
 # For importers, this is obligatory; the connectors will be used as the target for data updates
 enabled_connectors = ["localfile"]
 
+# Enable the AdminAPI importer
+[http.services.mentix.importers.adminapi]
+enabled_connectors = ["localfile"]
+# Should never allow access w/o prior authorization
+is_protected = true
+
 # Configure the Prometheus Service Discovery:
 [http.services.mentix.exporters.promsd]
 # The following files must be made available to Prometheus.

internal/http/services/mentix/mentix.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/cs3org/reva/pkg/mentix"
 	"github.com/cs3org/reva/pkg/mentix/config"
+	"github.com/cs3org/reva/pkg/mentix/exchangers"
 	"github.com/cs3org/reva/pkg/rhttp/global"
 )
 
@@ -63,13 +64,20 @@ func (s *svc) Unprotected() []string {
 	importers := s.mntx.GetRequestImporters()
 	exporters := s.mntx.GetRequestExporters()
 
-	endpoints := make([]string, len(importers)+len(exporters))
-	for idx, importer := range importers {
-		endpoints[idx] = importer.Endpoint()
-	}
-	for idx, exporter := range exporters {
-		endpoints[idx] = exporter.Endpoint()
+	getEndpoints := func(exchangers []exchangers.RequestExchanger) []string {
+		endpoints := make([]string, 0, len(exchangers))
+		for _, exchanger := range exchangers {
+			if !exchanger.IsProtectedEndpoint() {
+				endpoints = append(endpoints, exchanger.Endpoint())
+			}
+		}
+		return endpoints
 	}
+
+	endpoints := make([]string, 0, len(importers)+len(exporters))
+	endpoints = append(endpoints, getEndpoints(importers)...)
+	endpoints = append(endpoints, getEndpoints(exporters)...)
+
 	return endpoints
 }
 
@@ -130,7 +138,11 @@ func applyDefaultConfig(conf *config.Configuration) {
 
 	// Importers
 	if conf.Importers.WebAPI.Endpoint == "" {
-		conf.Importers.WebAPI.Endpoint = "/"
+		conf.Importers.WebAPI.Endpoint = "/sites"
+	}
+
+	if conf.Importers.AdminAPI.Endpoint == "" {
+		conf.Importers.AdminAPI.Endpoint = "/admin"
 	}
 
 	// Exporters
@@ -141,7 +153,7 @@ func applyDefaultConfig(conf *config.Configuration) {
 	}
 
 	if conf.Exporters.WebAPI.Endpoint == "" {
-		conf.Exporters.WebAPI.Endpoint = "/"
+		conf.Exporters.WebAPI.Endpoint = "/sites"
 	}
 	addDefaultConnector(&conf.Exporters.WebAPI.EnabledConnectors)
 

pkg/mentix/config/config.go

@@ -38,23 +38,33 @@ type Configuration struct {
 	Importers struct {
 		WebAPI struct {
 			Endpoint          string   `mapstructure:"endpoint"`
+			IsProtected       bool     `mapstructure:"is_protected"`
 			EnabledConnectors []string `mapstructure:"enabled_connectors"`
 		} `mapstructure:"webapi"`
+
+		AdminAPI struct {
+			Endpoint          string   `mapstructure:"endpoint"`
+			IsProtected       bool     `mapstructure:"is_protected"`
+			EnabledConnectors []string `mapstructure:"enabled_connectors"`
+		} `mapstructure:"adminapi"`
 	} `mapstructure:"importers"`
 
 	Exporters struct {
 		WebAPI struct {
 			Endpoint          string   `mapstructure:"endpoint"`
+			IsProtected       bool     `mapstructure:"is_protected"`
 			EnabledConnectors []string `mapstructure:"enabled_connectors"`
 		} `mapstructure:"webapi"`
 
 		CS3API struct {
 			Endpoint          string   `mapstructure:"endpoint"`
+			IsProtected       bool     `mapstructure:"is_protected"`
 			EnabledConnectors []string `mapstructure:"enabled_connectors"`
 		} `mapstructure:"cs3api"`
 
 		SiteLocations struct {
 			Endpoint          string   `mapstructure:"endpoint"`
+			IsProtected       bool     `mapstructure:"is_protected"`
 			EnabledConnectors []string `mapstructure:"enabled_connectors"`
 		} `mapstructure:"siteloc"`
 

pkg/mentix/config/ids.go

@@ -28,6 +28,8 @@ const (
 const (
 	// ImporterIDWebAPI is the identifier for the WebAPI importer.
 	ImporterIDWebAPI = "webapi"
+	// ImporterIDAdminAPI is the identifier for the AdminAPI importer.
+	ImporterIDAdminAPI = "adminapi"
 )
 
 const (

pkg/mentix/connectors/gocdb.go

@@ -75,6 +75,7 @@ func (connector *GOCDBConnector) RetrieveMeshData() (*meshdata.MeshData, error)
 		}
 	}
 
+	meshData.InferMissingData()
 	return meshData, nil
 }
 
@@ -127,6 +128,11 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData) error {
 	for _, site := range sites.Sites {
 		properties := connector.extensionsToMap(&site.Extensions)
 
+		// Sites coming from the GOCDB are always authorized by default
+		if value := meshdata.GetPropertyValue(properties, meshdata.PropertyAuthorized, ""); len(value) == 0 {
+			meshdata.SetPropertyValue(&properties, meshdata.PropertyAuthorized, "true")
+		}
+
 		// See if an organization has been defined using properties; otherwise, use the official name
 		organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName)
 

pkg/mentix/connectors/localfile.go

@@ -78,6 +78,7 @@ func (connector *LocalFileConnector) RetrieveMeshData() (*meshdata.MeshData, err
 	// Update the site types, as these are not part of the JSON data
 	connector.setSiteTypes(meshData)
 
+	meshData.InferMissingData()
 	return meshData, nil
 }
 
@@ -89,12 +90,23 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa
 		meshData = &meshdata.MeshData{}
 	}
 
-	if (updatedData.Flags & meshdata.FlagObsolete) == meshdata.FlagObsolete {
-		// Remove data by unmerging
-		meshData.Unmerge(updatedData)
-	} else {
-		// Add/update data by merging
-		meshData.Merge(updatedData)
+	err = nil
+	switch updatedData.Status {
+	case meshdata.StatusDefault:
+		err = connector.mergeData(meshData, updatedData)
+
+	case meshdata.StatusObsolete:
+		err = connector.unmergeData(meshData, updatedData)
+
+	case meshdata.StatusAuthorize:
+		err = connector.authorizeData(meshData, updatedData, true)
+
+	case meshdata.StatusUnauthorize:
+		err = connector.authorizeData(meshData, updatedData, false)
+	}
+
+	if err != nil {
+		return err
 	}
 
 	// Write the updated sites back to the file
@@ -106,6 +118,48 @@ func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshDa
 	return nil
 }
 
+func (connector *LocalFileConnector) mergeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData) error {
+	// Store the previous authorization status for already existing sites
+	siteAuthorizationStatus := make(map[string]string)
+	for _, site := range meshData.Sites {
+		siteAuthorizationStatus[site.ID] = meshdata.GetPropertyValue(site.Properties, meshdata.PropertyAuthorized, "false")
+	}
+
+	// Add/update data by merging
+	meshData.Merge(updatedData)
+
+	// Restore the authorization status for all sites
+	for siteID, status := range siteAuthorizationStatus {
+		if site := meshData.FindSite(siteID); site != nil {
+			meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, status)
+		}
+	}
+	return nil
+}
+
+func (connector *LocalFileConnector) unmergeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData) error {
+	// Remove data by unmerging
+	meshData.Unmerge(updatedData)
+	return nil
+}
+
+func (connector *LocalFileConnector) authorizeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData, authorize bool) error {
+	for _, placeholderSite := range updatedData.Sites {
+		// The site ID is stored in the updated site's name
+		if site := meshData.FindSite(placeholderSite.Name); site != nil {
+			if authorize {
+				meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "true")
+			} else {
+				meshdata.SetPropertyValue(&site.Properties, meshdata.PropertyAuthorized, "false")
+			}
+		} else {
+			return fmt.Errorf("no site with id '%v' found", placeholderSite.Name)
+		}
+	}
+
+	return nil
+}
+
 func (connector *LocalFileConnector) setSiteTypes(meshData *meshdata.MeshData) {
 	for _, site := range meshData.Sites {
 		site.Type = meshdata.SiteTypeCommunity // Sites coming from a local file are always community sites

pkg/mentix/exchangers/exporters/cs3api.go

@@ -37,7 +37,7 @@ func (exporter *CS3APIExporter) Activate(conf *config.Configuration, log *zerolo
 	}
 
 	// Store CS3API specifics
-	exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint)
+	exporter.SetEndpoint(conf.Exporters.CS3API.Endpoint, conf.Exporters.CS3API.IsProtected)
 	exporter.SetEnabledConnectors(conf.Exporters.CS3API.EnabledConnectors)
 
 	exporter.defaultActionHandler = cs3api.HandleDefaultQuery