Merge pull request #163 from joselsegura/show_report_result_in_metric

Bug 1882361: Get report from smart-proxy and expose overview as a metric

config/pod.yaml

@@ -6,3 +6,8 @@ interval: "2h"
 storagePath: /var/lib/insights-operator
 endpoint: https://cloud.redhat.com/api/ingress/v1/upload
 impersonate: system:serviceaccount:openshift-insights:gather
+pull_report:
+  endpoint: https://cloud.redhat.com/api/insights-results-aggregator/v1/clusters/%s/report
+  delay: "60s"
+  timeout: "3000s"
+  min_retry: "30s"

pkg/cmd/start/start.go

@@ -22,9 +22,13 @@ const serviceCACertPath = "/var/run/configmaps/service-ca-bundle/service-ca.crt"
 func NewOperator() *cobra.Command {
 	operator := &controller.Support{
 		Controller: config.Controller{
-			StoragePath: "/var/lib/insights-operator",
-			Interval:    10 * time.Minute,
-			Endpoint:    "https://cloud.redhat.com/api/ingress/v1/upload",
+			StoragePath:          "/var/lib/insights-operator",
+			Interval:             10 * time.Minute,
+			Endpoint:             "https://cloud.redhat.com/api/ingress/v1/upload",
+			ReportEndpoint:       "https://cloud.redhat.com/api/insights-results-aggregator/v1/cluster/%s/report",
+			ReportPullingDelay:   60 * time.Second,
+			ReportMinRetryTime:   10 * time.Second,
+			ReportPullingTimeout: 30 * time.Minute,
 		},
 	}
 	cfg := controllercmd.NewControllerCommandConfig("openshift-insights-operator", version.Get(), operator.Run)

pkg/config/config.go

@@ -5,21 +5,28 @@ import (
 	"time"
 )
 
-// Controller defines the standard config for this operator.
+// Serialized defines the standard config for this operator.
 type Serialized struct {
 	Report      bool   `json:"report"`
 	StoragePath string `json:"storagePath"`
 	Interval    string `json:"interval"`
 	Endpoint    string `json:"endpoint"`
+	PullReport  struct {
+		Endpoint     string `json:"endpoint"`
+		Delay        string `json:"delay"`
+		Timeout      string `json:"timeout"`
+		MinRetryTime string `json:"min_retry"`
+	} `json:"pull_report"`
 	Impersonate string `json:"impersonate"`
 }
 
 func (s *Serialized) ToController() (*Controller, error) {
 	cfg := Controller{
-		Report:      s.Report,
-		StoragePath: s.StoragePath,
-		Endpoint:    s.Endpoint,
-		Impersonate: s.Impersonate,
+		Report:         s.Report,
+		StoragePath:    s.StoragePath,
+		Endpoint:       s.Endpoint,
+		ReportEndpoint: s.PullReport.Endpoint,
+		Impersonate:    s.Impersonate,
 	}
 	if len(s.Interval) > 0 {
 		d, err := time.ParseDuration(s.Interval)
@@ -32,6 +39,43 @@ func (s *Serialized) ToController() (*Controller, error) {
 	if cfg.Interval <= 0 {
 		return nil, fmt.Errorf("interval must be a non-negative duration")
 	}
+
+	if len(s.PullReport.Delay) > 0 {
+		d, err := time.ParseDuration(s.PullReport.Delay)
+		if err != nil {
+			return nil, fmt.Errorf("delay must be a valid duration: %v", err)
+		}
+		cfg.ReportPullingDelay = d
+	}
+
+	if cfg.ReportPullingDelay <= 0 {
+		return nil, fmt.Errorf("delay must be a non-negative duration")
+	}
+
+	if len(s.PullReport.MinRetryTime) > 0 {
+		d, err := time.ParseDuration(s.PullReport.MinRetryTime)
+		if err != nil {
+			return nil, fmt.Errorf("min_retry must be a valid duration: %v", err)
+		}
+		cfg.ReportMinRetryTime = d
+	}
+
+	if cfg.ReportMinRetryTime <= 0 {
+		return nil, fmt.Errorf("min_retry must be a non-negative duration")
+	}
+
+	if len(s.PullReport.Timeout) > 0 {
+		d, err := time.ParseDuration(s.PullReport.Timeout)
+		if err != nil {
+			return nil, fmt.Errorf("timeout must be a valid duration: %v", err)
+		}
+		cfg.ReportPullingTimeout = d
+	}
+
+	if cfg.ReportPullingTimeout <= 0 {
+		return nil, fmt.Errorf("timeout must be a non-negative duration")
+	}
+
 	if len(cfg.StoragePath) == 0 {
 		return nil, fmt.Errorf("storagePath must point to a directory where snapshots can be stored")
 	}
@@ -40,11 +84,15 @@ func (s *Serialized) ToController() (*Controller, error) {
 
 // Controller defines the standard config for this operator.
 type Controller struct {
-	Report      bool
-	StoragePath string
-	Interval    time.Duration
-	Endpoint    string
-	Impersonate string
+	Report               bool
+	StoragePath          string
+	Interval             time.Duration
+	Endpoint             string
+	ReportEndpoint       string
+	ReportPullingDelay   time.Duration
+	ReportMinRetryTime   time.Duration
+	ReportPullingTimeout time.Duration
+	Impersonate          string
 
 	Username string
 	Password string

pkg/config/configobserver/configobserver.go

@@ -139,6 +139,41 @@ func (c *Controller) retrieveConfig(ctx context.Context) error {
 		if noproxy, ok := secret.Data["noProxy"]; ok {
 			nextConfig.HTTPConfig.NoProxy = string(noproxy)
 		}
+		if reportEndpoint, ok := secret.Data["reportEndpoint"]; ok {
+			nextConfig.ReportEndpoint = string(reportEndpoint)
+		}
+		if reportPullingDelay, ok := secret.Data["reportPullingDelay"]; ok {
+			if v, err := time.ParseDuration(string(reportPullingDelay)); err == nil {
+				nextConfig.ReportPullingDelay = v
+			} else {
+				klog.Warningf(
+					"reportPullingDelay secret contains an invalid value (%s). Using previous value",
+					reportPullingDelay,
+				)
+			}
+		} else {
+			nextConfig.ReportPullingDelay = time.Duration(-1)
+		}
+		if reportPullingTimeout, ok := secret.Data["reportPullingTimeout"]; ok {
+			if v, err := time.ParseDuration(string(reportPullingTimeout)); err == nil {
+				nextConfig.ReportPullingTimeout = v
+			} else {
+				klog.Warningf(
+					"reportPullingTimeout secret contains an invalid value (%s). Using previous value",
+					reportPullingTimeout,
+				)
+			}
+		}
+		if reportMinRetryTime, ok := secret.Data["reportMinRetryTime"]; ok {
+			if v, err := time.ParseDuration(string(reportMinRetryTime)); err == nil {
+				nextConfig.ReportMinRetryTime = v
+			} else {
+				klog.Warningf(
+					"reportMinRetryTime secret contains an invalid value (%s). Using previous value",
+					reportMinRetryTime,
+				)
+			}
+		}
 		nextConfig.Report = len(nextConfig.Endpoint) > 0
 
 		if intervalString, ok := secret.Data["interval"]; ok {
@@ -216,6 +251,18 @@ func (c *Controller) mergeConfigLocked() {
 		if len(c.secretConfig.Endpoint) > 0 {
 			cfg.Endpoint = c.secretConfig.Endpoint
 		}
+		if len(c.secretConfig.ReportEndpoint) > 0 {
+			cfg.ReportEndpoint = c.secretConfig.ReportEndpoint
+		}
+		if c.secretConfig.ReportPullingDelay >= 0 {
+			cfg.ReportPullingDelay = c.secretConfig.ReportPullingDelay
+		}
+		if c.secretConfig.ReportPullingTimeout > 0 {
+			cfg.ReportPullingTimeout = c.secretConfig.ReportPullingTimeout
+		}
+		if c.secretConfig.ReportMinRetryTime > 0 {
+			cfg.ReportMinRetryTime = c.secretConfig.ReportMinRetryTime
+		}
 		cfg.HTTPConfig = c.secretConfig.HTTPConfig
 	}
 	if c.tokenConfig != nil {
@@ -228,7 +275,10 @@ func (c *Controller) mergeConfigLocked() {
 func (c *Controller) setConfigLocked(config *config.Controller) {
 	if c.config != nil {
 		if !reflect.DeepEqual(c.config, config) {
-			klog.V(2).Infof("Configuration updated: enabled=%t endpoint=%s interval=%s username=%t token=%t", config.Report, config.Endpoint, config.Interval, len(config.Username) > 0, len(config.Token) > 0)
+			klog.V(2).Infof(
+				"Configuration updated: enabled=%t endpoint=%s interval=%s username=%t token=%t reportEndpoint=%s initialPollingDelay=%s minRetryTime=%s pollingTimeout=%s",
+				config.Report, config.Endpoint, config.Interval, len(config.Username) > 0, len(config.Token) > 0, config.ReportEndpoint,
+				config.ReportPullingDelay, config.ReportMinRetryTime, config.ReportPullingTimeout)
 			for _, ch := range c.listeners {
 				if ch == nil {
 					continue
@@ -240,7 +290,10 @@ func (c *Controller) setConfigLocked(config *config.Controller) {
 			}
 		}
 	} else {
-		klog.V(2).Infof("Configuration set: enabled=%t endpoint=%s interval=%s username=%t token=%t", config.Report, config.Endpoint, config.Interval, len(config.Username) > 0, len(config.Token) > 0)
+		klog.V(2).Infof(
+			"Configuration set: enabled=%t endpoint=%s interval=%s username=%t token=%t reportEndpoint=%s initialPollingDelay=%s minRetryTime=%s pollingTimeout=%s",
+			config.Report, config.Endpoint, config.Interval, len(config.Username) > 0, len(config.Token) > 0, config.ReportEndpoint,
+			config.ReportPullingDelay, config.ReportMinRetryTime, config.ReportPullingTimeout)
 	}
 	c.config = config
 }

pkg/controller/operator.go

@@ -32,6 +32,7 @@ import (
 	"github.com/openshift/insights-operator/pkg/gather"
 	"github.com/openshift/insights-operator/pkg/gather/clusterconfig"
 	"github.com/openshift/insights-operator/pkg/insights/insightsclient"
+	"github.com/openshift/insights-operator/pkg/insights/insightsreport"
 	"github.com/openshift/insights-operator/pkg/insights/insightsuploader"
 	"github.com/openshift/insights-operator/pkg/record/diskrecorder"
 )
@@ -46,6 +47,7 @@ func (s *Support) LoadConfig(obj map[string]interface{}) error {
 	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj, &cfg); err != nil {
 		return fmt.Errorf("unable to load config: %v", err)
 	}
+
 	controller, err := cfg.ToController()
 	if err != nil {
 		return err
@@ -54,7 +56,6 @@ func (s *Support) LoadConfig(obj map[string]interface{}) error {
 
 	data, _ := json.Marshal(cfg)
 	klog.V(2).Infof("Current config: %s", string(data))
-
 	return nil
 }
 
@@ -191,6 +192,9 @@ func (s *Support) Run(ctx context.Context, controller *controllercmd.ControllerC
 	// know any previous last reported time
 	go uploader.Run(ctx)
 
+	reportGatherer := insightsreport.New(insightsClient, configObserver, uploader)
+	go reportGatherer.Run(ctx)
+
 	klog.Warning("stopped")
 
 	<-ctx.Done()

pkg/controllerstatus/controllerstatus.go

@@ -13,8 +13,13 @@ type Interface interface {
 
 type Operation string
 
-// Specific flag for summary related to uploading process.
-const Uploading Operation = "Uploading"
+
+const (
+	// DownloadingReport specific flag for Smart Proxy report downloading process.
+	DownloadingReport Operation = "DownloadingReport"
+	// Uploading specific flag for summary related to uploading process.
+	Uploading Operation = "Uploading"
+)
 
 type Summary struct {
 	Operation          Operation