xcloud metrics gathering (#1403)

changelog/unreleased/xcloud-metrics.md

@@ -0,0 +1,7 @@
+Enhancement: support remote cloud gathering metrics
+
+The current metrics package can only gather metrics either from
+json files. With this feature, the metrics can be gathered polling
+the http endpoints exposed by the owncloud/nextcloud sciencemesh apps.
+
+https://github.com/cs3org/reva/pull/1403

examples/metrics/xcloud.toml

@@ -0,0 +1,11 @@
+[http]
+address = "0.0.0.0:5550"
+
+[http.services.metrics]
+metrics_data_driver_type = "xcloud"
+metrics_record_interval = 5000
+xcloud_instance = "http://localhost"
+xcloud_interval = 5
+xcloud_catalog = 'https://sciencemesh-test.uni-muenster.de/api/mentix/sites?action=register'
+
+[http.services.prometheus]

pkg/metrics/config/config.go

@@ -23,6 +23,9 @@ type Config struct {
  MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"`
  MetricsDataLocation string `mapstructure:"metrics_data_location"`
  MetricsRecordInterval int `mapstructure:"metrics_record_interval"`
+ XcloudInstance string `mapstructure:"xcloud_instance"`
+ XcloudCatalog string `mapstructure:"xcloud_catalog"`
+ XcloudPullInterval int `mapstructure:"xcloud_pull_interval"`
 }
 
 // Init sets sane defaults
@@ -33,6 +36,7 @@ func (c *Config) Init() {
  c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json"
  }
  }
+
  if c.MetricsRecordInterval == 0 {
  c.MetricsRecordInterval = 5000
  }

pkg/metrics/driver/loader/loader.go

@@ -22,5 +22,6 @@ import (
  // Load metrics drivers.
  _ "github.com/cs3org/reva/pkg/metrics/driver/dummy"
  _ "github.com/cs3org/reva/pkg/metrics/driver/json"
+ _ "github.com/cs3org/reva/pkg/metrics/driver/xcloud"
  // Add your own here
 )

pkg/metrics/driver/xcloud/xcloud.go

@@ -0,0 +1,269 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package json
+
+import (
+ "bytes"
+ "crypto/tls"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "sync"
+ "time"
+
+ "github.com/cs3org/reva/pkg/metrics/driver/registry"
+
+ "github.com/cs3org/reva/pkg/logger"
+ "github.com/cs3org/reva/pkg/metrics/config"
+ "github.com/rs/zerolog"
+)
+
+var log zerolog.Logger
+
+func init() {
+ log = logger.New().With().Int("pid", os.Getpid()).Logger()
+ driver := &CloudDriver{CloudData: &CloudData{}}
+ registry.Register(driverName(), driver)
+
+}
+
+func driverName() string {
+ return "xcloud"
+}
+
+// CloudDriver is the driver to use for Sciencemesh apps
+type CloudDriver struct {
+ instance string
+ catalog string
+ pullInterval int
+ CloudData *CloudData
+ sync.Mutex
+ client *http.Client
+}
+
+func (d *CloudDriver) refresh() error {
+
+ // endpoint example: https://mybox.com or https://mybox.com/owncloud
+ endpoint := fmt.Sprintf("%s/index.php/apps/sciencemesh/internal_metrics", d.instance)
+
+ req, err := http.NewRequest("GET", endpoint, nil)
+ if err != nil {
+ log.Err(err).Msgf("xcloud: error creating request to %s", d.instance)
+ return err
+ }
+
+ resp, err := d.client.Do(req)
+ if err != nil {
+ log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance)
+ return err
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ err := fmt.Errorf("xcloud: error getting internal metrics from %s. http status code (%d)", d.instance, resp.StatusCode)
+ log.Err(err).Msgf("xcloud: error getting internal metrics from %s", d.instance)
+ return err
+ }
+ defer resp.Body.Close()
+
+ // read response body
+ data, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ log.Err(err).Msgf("xcloud: error reading resp body from internal metrics from %s", d.instance)
+ return err
+ }
+
+ cd := &CloudData{}
+ if err := json.Unmarshal(data, cd); err != nil {
+ log.Err(err).Msgf("xcloud: error parsing body from internal metrics: body(%s)", string(data))
+ return err
+ }
+
+ d.Lock()
+ defer d.Unlock()
+ d.CloudData = cd
+ log.Info().Msgf("xcloud: received internal metrics from cloud provider: %+v", cd)
+
+ mc := &MentixCatalog{
+ Name: cd.Settings.Sitename,
+ FullName: cd.Settings.Sitename,
+ Homepage: cd.Settings.Hostname,
+ Description: "ScienceMesh App from " + cd.Settings.Sitename,
+ CountryCode: cd.Settings.Country,
+ Services: []*MentixService{
+ &MentixService{
+ Host: cd.Settings.Hostname,
+ IsMonitored: true,
+ Name: cd.Settings.Hostname + " - REVAD",
+ URL: cd.Settings.Siteurl,
+ Properties: &MentixServiceProperties{
+ MetricsPath: "/index.php/apps/sciencemesh/metrics",
+ },
+ Type: &MentixServiceType{
+ Name: "REVAD",
+ },
+ },
+ },
+ }
+
+ j, err := json.Marshal(mc)
+ if err != nil {
+ log.Err(err).Msgf("xcloud: error marhsaling mentix calalog info")
+ return err
+ }
+
+ log.Info().Msgf("xcloud: info to send to register: %s", string(j))
+
+ // send to register if catalog is set
+ req, err = http.NewRequest("POST", d.catalog, bytes.NewBuffer(j))
+ if err != nil {
+ log.Err(err).Msgf("xcloud: error creating POST request to: %s", d.catalog)
+ return err
+ }
+
+ resp, err = d.client.Do(req)
+ if err != nil {
+ log.Err(err).Msgf("xcloud: error registering catalog info to: %s with info: %s", d.catalog, string(j))
+ return err
+ }
+
+ defer resp.Body.Close()
+ body, _ := ioutil.ReadAll(resp.Body)
+
+ if resp.StatusCode != http.StatusOK {
+ err := fmt.Errorf("xcloud: error registering site: status code(%d) body(%s)", resp.StatusCode, string(body))
+ log.Err(err).Msg("xcloud: error registering site")
+ return err
+ }
+
+ log.Info().Msgf("xcloud: site registered: %s", string(body))
+ return nil
+}
+
+// Configure configures this driver
+func (d *CloudDriver) Configure(c *config.Config) error {
+ if c.XcloudInstance == "" {
+ err := errors.New("xcloud: missing xcloud_instance config parameter")
+ return err
+ }
+
+ if c.XcloudPullInterval == 0 {
+ c.XcloudPullInterval = 10 // seconds
+ }
+
+ d.instance = c.XcloudInstance
+ d.pullInterval = c.XcloudPullInterval
+ d.catalog = c.XcloudCatalog
+
+ // TODO(labkode): make it configurable once site adopted are prod-ready
+ tr := &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ }
+ client := &http.Client{Transport: tr}
+
+ d.client = client
+
+ ticker := time.NewTicker(time.Duration(d.pullInterval) * time.Second)
+ quit := make(chan struct{})
+ go func() {
+ for {
+ select {
+ case <-ticker.C:
+ err := d.refresh()
+ if err != nil {
+ log.Err(err).Msgf("xcloud: error from refresh goroutine")
+ }
+ case <-quit:
+ ticker.Stop()
+ return
+ }
+ }
+ }()
+
+ return nil
+}
+
+// GetNumUsers returns the number of site users
+func (d *CloudDriver) GetNumUsers() int64 {
+ return d.CloudData.Metrics.TotalUsers
+}
+
+// GetNumGroups returns the number of site groups
+func (d *CloudDriver) GetNumGroups() int64 {
+ return d.CloudData.Metrics.TotalGroups
+}
+
+// GetAmountStorage returns the amount of site storage used
+func (d *CloudDriver) GetAmountStorage() int64 {
+ return d.CloudData.Metrics.TotalStorage
+}
+
+// CloudData represents the information obtained from the sciencemesh app
+type CloudData struct {
+ Metrics CloudDataMetrics `json:"metrics"`
+ Settings CloudDataSettings `json:"settings"`
+}
+
+// CloudDataMetrics reprents the metrics gathered from the sciencemesh app
+type CloudDataMetrics struct {
+ TotalUsers int64 `json:"numusers"`
+ TotalGroups int64 `json:"numgroups"`
+ TotalStorage int64 `json:"numstorage"`
+}
+
+// CloudDataSettings represents the metrics gathered
+type CloudDataSettings struct {
+ IOPUrl string `json:"iopurl"`
+ Sitename string `json:"sitename"`
+ Siteurl string `json:"siteurl"`
+ Hostname string `json:"hostname"`
+ Country string `json:"country"`
+}
+
+// MentixCatalog represents the information needed to register a site into the mesh
+type MentixCatalog struct {
+ CountryCode string `json:"CountryCode"`
+ Description string `json:"Description"`
+ FullName string `json:"FullName"`
+ Homepage string `json:"Homepage"`
+ Name string `json:"Name"`
+ Services []*MentixService `json:"Services"`
+}
+
+// MentixService represents the service running in a site
+type MentixService struct {
+ Host string `json:"Host"`
+ IsMonitored bool `json:"IsMonitored"`
+ Name string `json:"Name"`
+ Properties *MentixServiceProperties `json:"Properties"`
+ Type *MentixServiceType `json:"Type"`
+ URL string `json:"URL"`
+}
+
+// MentixServiceProperties represents the properties to expose the metrics endpoint
+type MentixServiceProperties struct {
+ MetricsPath string `json:"METRICS_PATH"`
+}
+
+// MentixServiceType represents the type of service running
+type MentixServiceType struct {
+ Name string `json:"Name"`
+}