ApiRequestCount conditional gathering

docs/gathered-data.md

@@ -1,5 +1,17 @@
 This document is auto-generated by `make docs`
 
+## APIRequestCounts
+
+BuildGatherApiRequestCounts creates a gathering closure which collects API requests counts for the
+resources mentioned in the alert provided as a string parameter
+Params is of type AlertIsFiringConditionParams:
+  - alert_name string - name of the firing alert
+
+* Location in archive: conditional/alerts/<alert_name>/api_request_counts.json
+* Since versions:
+  * 4.10+
+
+
 ## CRD
 
 collects the specified Custom Resource Definitions.

docs/insights-archive-sample/conditional/api_request_counts.json

@@ -0,0 +1,8 @@
+[
+    {
+        "resource": "customresourcedefinitions.v1beta1.apiextensions.k8s.io",
+        "removed_in_release": "1.22",
+        "total_request_count": 46,
+        "last_day_request_count": 0
+    }
+]

manifests/03-clusterrole.yaml

@@ -212,6 +212,14 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - apiserver.openshift.io
+    resources:
+      - apirequestcounts
+    verbs:
+      - get
+      - list
+      - watch  
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

pkg/gatherers/conditional/conditional_gatherer.go

@@ -27,6 +27,7 @@ import (
 var gatheringFunctionBuilders = map[GatheringFunctionName]GathererFunctionBuilderPtr{
 	GatherLogsOfNamespace:         (*Gatherer).BuildGatherLogsOfNamespace,
 	GatherImageStreamsOfNamespace: (*Gatherer).BuildGatherImageStreamsOfNamespace,
+	GatherAPIRequestCounts:        (*Gatherer).BuildGatherAPIRequestCounts,
 }
 
 // gatheringRules contains all the rules used to run conditional gatherings.
@@ -77,6 +78,21 @@ var defaultGatheringRules = []GatheringRule{
 			},
 		},
 	},
+	{
+		Conditions: []ConditionWithParams{
+			{
+				Type: AlertIsFiring,
+				Params: AlertIsFiringConditionParams{
+					Name: "APIRemovedInNextEUSReleaseInUse",
+				},
+			},
+		},
+		GatheringFunctions: GatheringFunctions{
+			GatherAPIRequestCounts: GatherAPIRequestCountsParams{
+				AlertName: "APIRemovedInNextEUSReleaseInUse",
+			},
+		},
+	},
 }
 
 const canConditionalGathererFail = false
@@ -86,8 +102,9 @@ type Gatherer struct {
 	gatherProtoKubeConfig   *rest.Config
 	metricsGatherKubeConfig *rest.Config
 	imageKubeConfig         *rest.Config
-	firingAlerts            map[string]bool // golang doesn't have sets :(
-	gatheringRules          []GatheringRule
+	// there can be multiple instances of the same alert
+	firingAlerts   map[string][]AlertLabels
+	gatheringRules []GatheringRule
 }
 
 // New creates a new instance of conditional gatherer with the appropriate configs
@@ -138,7 +155,6 @@ func (g *Gatherer) GetGatheringFunctions(ctx context.Context) (map[string]gather
 		if err != nil {
 			return nil, err
 		}
-
 		if allConditionsAreSatisfied {
 			functions, errs := g.createGatheringClosures(conditionalGathering.GatheringFunctions)
 			if len(errs) > 0 {
@@ -206,7 +222,7 @@ func (g *Gatherer) updateAlertsCache(ctx context.Context) error {
 func (g *Gatherer) updateAlertsCacheFromClient(ctx context.Context, metricsClient rest.Interface) error {
 	const logPrefix = "conditional gatherer: "
 
-	g.firingAlerts = make(map[string]bool)
+	g.firingAlerts = make(map[string][]AlertLabels)
 
 	data, err := metricsClient.Get().AbsPath("federate").
 		Param("match[]", `ALERTS{alertstate="firing"}`).
@@ -237,17 +253,20 @@ func (g *Gatherer) updateAlertsCacheFromClient(ctx context.Context, metricsClien
 			klog.Info(logPrefix + "metric is nil")
 			continue
 		}
-
+		alertLabels := make(map[string]string)
 		for _, label := range metric.GetLabel() {
 			if label == nil {
 				klog.Info(logPrefix + "label is nil")
 				continue
 			}
-
-			if label.GetName() == "alertname" {
-				g.firingAlerts[label.GetValue()] = true
-			}
+			alertLabels[label.GetName()] = label.GetValue()
+		}
+		alertName, ok := alertLabels["alertname"]
+		if !ok {
+			klog.Warningf("%s can't find \"alertname\" label in the metric: %v", logPrefix, metric)
+			continue
 		}
+		g.firingAlerts[alertName] = append(g.firingAlerts[alertName], alertLabels)
 	}
 
 	return nil

pkg/gatherers/conditional/conditional_gatherer_test.go

@@ -166,7 +166,7 @@ func Test_Gatherer_GatherConditionalGathererRules(t *testing.T) {
 	err = json.Unmarshal(item, &gotGatheringRules)
 	assert.NoError(t, err)
 
-	assert.Len(t, gotGatheringRules, 1)
+	assert.Len(t, gotGatheringRules, 2)
 }
 
 func newFakeClientWithMetrics(metrics string) *fake.RESTClient {

pkg/gatherers/conditional/gather_api_request_count.go

@@ -0,0 +1,107 @@
+package conditional
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/openshift/insights-operator/pkg/gatherers"
+	"github.com/openshift/insights-operator/pkg/record"
+	"github.com/openshift/insights-operator/pkg/utils"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+)
+
+// APIRequestCount defines a type used when marshaling into JSON
+type APIRequestCount struct {
+	ResourceName        string `json:"resource"`
+	RemovedInRelease    string `json:"removed_in_release"`
+	TotalRequestCount   int64  `json:"total_request_count"`
+	LastDayRequestCount int64  `json:"last_day_request_count"`
+}
+
+// BuildGatherApiRequestCounts creates a gathering closure which collects API requests counts for the
+// resources mentioned in the alert provided as a string parameter
+// Params is of type AlertIsFiringConditionParams:
+//   - alert_name string - name of the firing alert
+//
+// * Location in archive: conditional/alerts/<alert_name>/api_request_counts.json
+// * Since versions:
+//   * 4.10+
+func (g *Gatherer) BuildGatherAPIRequestCounts(paramsInterface interface{}) (gatherers.GatheringClosure, error) {
+	params, ok := paramsInterface.(GatherAPIRequestCountsParams)
+	if !ok {
+		return gatherers.GatheringClosure{}, fmt.Errorf(
+			"unexpected type in paramsInterface, expected %T, got %T",
+			GatherAPIRequestCountsParams{}, paramsInterface,
+		)
+	}
+
+	return gatherers.GatheringClosure{
+		Run: func(ctx context.Context) ([]record.Record, []error) {
+			dynamicClient, err := dynamic.NewForConfig(g.gatherProtoKubeConfig)
+			if err != nil {
+				return nil, []error{err}
+			}
+			records, errs := g.gatherAPIRequestCounts(ctx, dynamicClient, params.AlertName)
+			if errs != nil {
+				return records, errs
+			}
+			return records, nil
+		},
+		CanFail: canConditionalGathererFail,
+	}, nil
+}
+
+func (g *Gatherer) gatherAPIRequestCounts(ctx context.Context,
+	dynamicClient dynamic.Interface, alertName string) ([]record.Record, []error) {
+	resources := make(map[string]struct{})
+	for _, labels := range g.firingAlerts[alertName] {
+		resourceName := fmt.Sprintf("%s.%s.%s", labels["resource"], labels["version"], labels["group"])
+		resources[resourceName] = struct{}{}
+	}
+
+	gvr := schema.GroupVersionResource{Group: "apiserver.openshift.io", Version: "v1", Resource: "apirequestcounts"}
+	apiReqCountsList, err := dynamicClient.Resource(gvr).List(ctx, metav1.ListOptions{})
+	if errors.IsNotFound(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, []error{err}
+	}
+	var records []record.Record
+	var errrs []error
+	var apiReqCounts []APIRequestCount
+	for i := range apiReqCountsList.Items {
+		it := apiReqCountsList.Items[i]
+
+		// filter only resources we're interested in
+		if _, ok := resources[it.GetName()]; ok {
+			totalReqCount, err := utils.NestedInt64Wrapper(it.Object, "status", "requestCount")
+			if err != nil {
+				errrs = append(errrs, err)
+			}
+			lastDayReqCount, err := utils.NestedInt64Wrapper(it.Object, "status", "currentHour", "requestCount")
+			if err != nil {
+				errrs = append(errrs, err)
+			}
+			removedInRel, err := utils.NestedStringWrapper(it.Object, "status", "removedInRelease")
+			if err != nil {
+				errrs = append(errrs, err)
+			}
+			apiReqCount := APIRequestCount{
+				TotalRequestCount:   totalReqCount,
+				LastDayRequestCount: lastDayReqCount,
+				ResourceName:        it.GetName(),
+				RemovedInRelease:    removedInRel,
+			}
+			apiReqCounts = append(apiReqCounts, apiReqCount)
+		}
+	}
+	records = append(records, record.Record{
+		Name: fmt.Sprintf("%v/alerts/%s/api_request_counts", g.GetName(), alertName),
+		Item: record.JSONMarshaller{Object: apiReqCounts},
+	})
+	return records, errrs
+}

pkg/gatherers/conditional/gather_api_request_count_test.go

@@ -0,0 +1,97 @@
+package conditional
+
+import (
+	"context"
+	"testing"
+
+	"github.com/openshift/insights-operator/pkg/record"
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer/yaml"
+	"k8s.io/client-go/dynamic"
+	dynamicfake "k8s.io/client-go/dynamic/fake"
+)
+
+var apiRequestCountYAML1 = `
+apiVersion: apiserver.openshift.io/v1
+kind: APIRequestCount
+metadata:
+    name: test1.v1beta2.testapi.org
+status:
+    currentHour:
+        requestCount: 12
+    requestCount: 13
+    removedInRelease: "14.15"
+`
+
+var apiRequestCountYAML2 = `
+apiVersion: apiserver.openshift.io/v1
+kind: APIRequestCount
+metadata:
+    name: test2.v1beta1.testapi.org
+status:
+    currentHour:
+        requestCount: 2
+    requestCount: 3
+    removedInRelease: "1.123"
+`
+
+func Test_GatherAPIRequestCount(t *testing.T) {
+	gatherer := Gatherer{
+		firingAlerts: map[string][]AlertLabels{
+			"alertA": {
+				{
+					"alertname": "alertA",
+					"resource":  "test1",
+					"group":     "testapi.org",
+					"version":   "v1beta2",
+				},
+				{
+					"alertname": "alertA",
+					"resource":  "test2",
+					"group":     "testapi.org",
+					"version":   "v1beta1",
+				},
+			},
+		},
+	}
+
+	gvr := schema.GroupVersionResource{Group: "apiserver.openshift.io", Version: "v1", Resource: "apirequestcounts"}
+	client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{
+		gvr: "APIRequestCountsList",
+	})
+	createResource(t, apiRequestCountYAML1, client, gvr)
+	createResource(t, apiRequestCountYAML2, client, gvr)
+	records, errs := gatherer.gatherAPIRequestCounts(context.Background(), client, "alertA")
+	assert.Empty(t, errs, "Unexpected errors during gathering of API request counts")
+	assert.Len(t, records, 1, "Unexpected number of records")
+
+	// check gathered data
+	a, ok := records[0].Item.(record.JSONMarshaller).Object.([]APIRequestCount)
+	assert.True(t, ok, "Failed to convert")
+	assert.Len(t, a, 2, "Unexpected number of alerts")
+	assert.Equal(t, a[0].ResourceName, "test1.v1beta2.testapi.org")
+	assert.Equal(t, a[0].LastDayRequestCount, int64(12))
+	assert.Equal(t, a[0].TotalRequestCount, int64(13))
+	assert.Equal(t, a[0].RemovedInRelease, "14.15")
+
+	assert.Equal(t, a[1].ResourceName, "test2.v1beta1.testapi.org")
+	assert.Equal(t, a[1].LastDayRequestCount, int64(2))
+	assert.Equal(t, a[1].TotalRequestCount, int64(3))
+	assert.Equal(t, a[1].RemovedInRelease, "1.123")
+}
+
+func createResource(t *testing.T, yamlDef string, client dynamic.Interface, gvr schema.GroupVersionResource) {
+	decUnstructured := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
+
+	testAPIRequestCountObj := &unstructured.Unstructured{}
+
+	_, _, err := decUnstructured.Decode([]byte(yamlDef), nil, testAPIRequestCountObj)
+	assert.NoError(t, err, "Failed to decode API request count YAML definition")
+
+	_, err = client.Resource(gvr).Create(context.Background(), testAPIRequestCountObj, metav1.CreateOptions{})
+	assert.NoError(t, err, "Failed to create fake API request count resource")
+}

pkg/gatherers/conditional/gathering_functions.go

@@ -13,13 +13,19 @@ type GatheringFunctions = map[GatheringFunctionName]interface{}
 // GatheringFunctionName defines functions of conditional gatherer
 type GatheringFunctionName string
 
-// GatherLogsOfNamespace is a function collecting logs of the provided namespace.
-// See file gather_logs_of_namespace.go
-const GatherLogsOfNamespace GatheringFunctionName = "logs_of_namespace"
+const (
+	// GatherLogsOfNamespace is a function collecting logs of the provided namespace.
+	// See file gather_logs_of_namespace.go
+	GatherLogsOfNamespace GatheringFunctionName = "logs_of_namespace"
 
-// GatherImageStreamsOfNamespace is a function collecting image streams of the provided namespace.
-// See file gather_image_streams_of_namespace.go
-const GatherImageStreamsOfNamespace GatheringFunctionName = "image_streams_of_namespace"
+	// GatherImageStreamsOfNamespace is a function collecting image streams of the provided namespace.
+	// See file gather_image_streams_of_namespace.go
+	GatherImageStreamsOfNamespace GatheringFunctionName = "image_streams_of_namespace"
+
+	// GatherAPIRequestCounts is a function collecting api request counts for the resources read
+	// from the corresponding alert
+	GatherAPIRequestCounts GatheringFunctionName = "api_request_counts_of_resource_from_alert"
+)
 
 func (name GatheringFunctionName) NewParams(jsonParams []byte) (interface{}, error) {
 	switch name {
@@ -31,6 +37,10 @@ func (name GatheringFunctionName) NewParams(jsonParams []byte) (interface{}, err
 		var result GatherImageStreamsOfNamespaceParams
 		err := json.Unmarshal(jsonParams, &result)
 		return result, err
+	case GatherAPIRequestCounts:
+		var params GatherAPIRequestCountsParams
+		err := json.Unmarshal(jsonParams, &params)
+		return params, err
 	}
 	return nil, fmt.Errorf("unable to create params for %T: %v", name, name)
 }
@@ -50,3 +60,8 @@ type GatherImageStreamsOfNamespaceParams struct {
 	// Namespace from which to collect image streams
 	Namespace string `json:"namespace"`
 }
+
+// GatherAPIRequestCountsParams defines parameters for api_request_counts gatherer
+type GatherAPIRequestCountsParams struct {
+	AlertName string `json:"alert_name"`
+}