🤖 Sync from open-cluster-management-io/config-policy-controller: #77

Makefile

@@ -177,8 +177,8 @@ deploy: generate-operator-yaml
 
 .PHONY: create-ns
 create-ns:
- @kubectl create namespace $(CONTROLLER_NAMESPACE) || true
- @kubectl create namespace $(WATCH_NAMESPACE) || true
+ -@kubectl create namespace $(CONTROLLER_NAMESPACE)
+ -@kubectl create namespace $(WATCH_NAMESPACE)
 
 # Run against the current locally configured Kubernetes cluster
 .PHONY: run
@@ -239,7 +239,7 @@ kind-bootstrap-cluster-dev: kind-create-cluster install-crds install-resources
 .PHONY: kind-deploy-controller
 kind-deploy-controller: generate-operator-yaml
  @echo Installing $(IMG)
- kubectl create ns $(KIND_NAMESPACE) || true
+ -kubectl create ns $(KIND_NAMESPACE)
  kubectl apply -f deploy/operator.yaml -n $(KIND_NAMESPACE)
  kubectl patch deployment $(IMG) -n $(KIND_NAMESPACE) -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"$(IMG)\",\"env\":[{\"name\":\"WATCH_NAMESPACE\",\"value\":\"$(WATCH_NAMESPACE)\"}]}]}}}}"
 

controllers/configurationpolicy_controller.go

@@ -38,7 +38,6 @@ import (
  "k8s.io/kubectl/pkg/validation"
  ctrl "sigs.k8s.io/controller-runtime"
  "sigs.k8s.io/controller-runtime/pkg/client"
- "sigs.k8s.io/controller-runtime/pkg/metrics"
  "sigs.k8s.io/controller-runtime/pkg/reconcile"
 
  policyv1 "open-cluster-management.io/config-policy-controller/api/v1"
@@ -68,59 +67,6 @@ var (
  reasonCleanupError = "Error cleaning up child objects"
 )
 
-var evalLoopHistogram = prometheus.NewHistogram(
- prometheus.HistogramOpts{
- Name: "config_policies_evaluation_duration_seconds",
- Help: "The seconds that it takes to evaluate all configuration policies on the cluster",
- Buckets: []float64{1, 3, 9, 10.5, 15, 30, 60, 90, 120, 180, 300, 450, 600},
- },
-)
-
-var policyEvalSecondsCounter = prometheus.NewCounterVec(
- prometheus.CounterOpts{
- Name: "config_policy_evaluation_seconds_total",
- Help: "The total seconds taken while evaluating the configuration policy. Use this alongside " +
- "config_policy_evaluation_total.",
- },
- []string{"name"},
-)
-
-var policyEvalCounter = prometheus.NewCounterVec(
- prometheus.CounterOpts{
- Name: "config_policy_evaluation_total",
- Help: "The total number of evaluations of the configuration policy. Use this alongside " +
- "config_policy_evaluation_seconds_total.",
- },
- []string{"name"},
-)
-
-var compareObjSecondsCounter = prometheus.NewCounterVec(
- prometheus.CounterOpts{
- Name: "compare_objects_seconds_total",
- Help: "The total seconds taken while comparing policy objects. Use this alongside " +
- "compare_objects_evaluation_total.",
- },
- []string{"config_policy_name", "namespace", "object"},
-)
-
-var compareObjEvalCounter = prometheus.NewCounterVec(
- prometheus.CounterOpts{
- Name: "compare_objects_evaluation_total",
- Help: "The total number of times the comparison algorithm is run on an object. " +
- "Use this alongside compare_objects_seconds_total.",
- },
- []string{"config_policy_name", "namespace", "object"},
-)
-
-func init() {
- // Register custom metrics with the global Prometheus registry
- metrics.Registry.MustRegister(evalLoopHistogram)
- metrics.Registry.MustRegister(policyEvalSecondsCounter)
- metrics.Registry.MustRegister(policyEvalCounter)
- metrics.Registry.MustRegister(compareObjSecondsCounter)
- metrics.Registry.MustRegister(compareObjEvalCounter)
-}
-
 // SetupWithManager sets up the controller with the Manager.
 func (r *ConfigurationPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
  return ctrl.NewControllerManagedBy(mgr).
@@ -160,6 +106,8 @@ type ConfigurationPolicyReconciler struct {
  // where the controller is running.
  TargetK8sClient kubernetes.Interface
  TargetK8sConfig *rest.Config
+ // Whether custom metrics collection is enabled
+ EnableMetrics bool
  discoveryInfo
  // This is used to fetch and parse OpenAPI documents to perform client-side validation of object definitions.
  openAPIParser *openapi.CachedOpenAPIParser
@@ -181,6 +129,8 @@ func (r *ConfigurationPolicyReconciler) Reconcile(ctx context.Context, request c
  _ = policyEvalCounter.DeleteLabelValues(request.Name)
  _ = compareObjEvalCounter.DeletePartialMatch(prometheus.Labels{"config_policy_name": request.Name})
  _ = compareObjSecondsCounter.DeletePartialMatch(prometheus.Labels{"config_policy_name": request.Name})
+ _ = policyRelatedObjectGauge.DeletePartialMatch(
+ prometheus.Labels{"policy": fmt.Sprintf("%s/%s", request.Namespace, request.Name)})
  }
 
  return reconcile.Result{}, nil
@@ -234,6 +184,9 @@ func (r *ConfigurationPolicyReconciler) PeriodicallyExecConfigPolicies(freq uint
  if !skipLoop {
  log.Info("Processing the policies", "count", len(policiesList.Items))
 
+ // Initialize the related object map
+ policyRelatedObjectMap = sync.Map{}
+
  for i := 0; i < int(r.EvaluationConcurrency); i++ {
  wg.Add(1)
 
@@ -254,6 +207,11 @@ func (r *ConfigurationPolicyReconciler) PeriodicallyExecConfigPolicies(freq uint
  close(policyQueue)
  wg.Wait()
 
+ // Update the related object metric after policy processing
+ if r.EnableMetrics {
+ updateRelatedObjectMetric()
+ }
+ // Update the evaluation histogram with the elapsed time
  elapsed := time.Since(start).Seconds()
  evalLoopHistogram.Observe(elapsed)
  // making sure that if processing is > freq we don't sleep
@@ -973,14 +931,14 @@ func (r *ConfigurationPolicyReconciler) handleObjectTemplates(plc policyv1.Confi
 func (r *ConfigurationPolicyReconciler) checkRelatedAndUpdate(
  plc policyv1.ConfigurationPolicy, related, oldRelated []policyv1.RelatedObject, sendEvent bool,
 ) {
- sortRelatedObjectsAndUpdate(&plc, related, oldRelated)
+ sortRelatedObjectsAndUpdate(&plc, related, oldRelated, r.EnableMetrics)
  // An update always occurs to account for the lastEvaluated status field
  r.addForUpdate(&plc, sendEvent)
 }
 
 // helper function to check whether related objects has changed
 func sortRelatedObjectsAndUpdate(
- plc *policyv1.ConfigurationPolicy, related, oldRelated []policyv1.RelatedObject,
+ plc *policyv1.ConfigurationPolicy, related, oldRelated []policyv1.RelatedObject, collectMetrics bool,
 ) {
  sort.SliceStable(related, func(i, j int) bool {
  if related[i].Object.Kind != related[j].Object.Kind {
@@ -995,7 +953,33 @@ func sortRelatedObjectsAndUpdate(
 
  update := false
 
+ // Instantiate found objects for the related object metric
+ found := map[string]bool{}
+
+ if collectMetrics {
+ for _, obj := range oldRelated {
+ found[getObjectString(obj)] = false
+ }
+ }
+
+ // Format policy for related object metric
+ policyVal := fmt.Sprintf("%s/%s", plc.Namespace, plc.Name)
+
  for i, newEntry := range related {
+ var objKey string
+ // Collect the policy and related object for related object metric
+ if collectMetrics {
+ objKey = getObjectString(newEntry)
+ policiesArray := []string{}
+
+ if objValue, ok := policyRelatedObjectMap.Load(objKey); ok {
+ policiesArray = append(policiesArray, objValue.([]string)...)
+ }
+
+ policiesArray = append(policiesArray, policyVal)
+ policyRelatedObjectMap.Store(objKey, policiesArray)
+ }
+
  for _, oldEntry := range oldRelated {
  // Get matching objects
  if gocmp.Equal(newEntry.Object, oldEntry.Object) {
@@ -1005,11 +989,27 @@ func sortRelatedObjectsAndUpdate(
  !(*newEntry.Properties.CreatedByPolicy) {
  // Use the old properties if they existed and this is not a newly created resource
  related[i].Properties = oldEntry.Properties
+
+ if collectMetrics {
+ found[objKey] = true
+ }
+
+ break
  }
  }
  }
  }
 
+ // Clean up old related object metrics if the related object list changed
+ if collectMetrics {
+ for _, obj := range oldRelated {
+ objString := getObjectString(obj)
+ if !found[objString] {
+ _ = policyRelatedObjectGauge.DeleteLabelValues(objString, policyVal)
+ }
+ }
+ }
+
  if len(oldRelated) == len(related) {
  for i, entry := range oldRelated {
  if !gocmp.Equal(entry, related[i]) {

controllers/configurationpolicy_controller_test.go

@@ -412,14 +412,14 @@ func TestSortRelatedObjectsAndUpdate(t *testing.T) {
 
  empty := []policyv1.RelatedObject{}
 
- sortRelatedObjectsAndUpdate(policy, relatedList, empty)
+ sortRelatedObjectsAndUpdate(policy, relatedList, empty, false)
  assert.True(t, relatedList[0].Object.Metadata.Name == "bar")
 
  // append another object named bar but also with namespace bar
  relatedList = append(relatedList, addRelatedObjects(true, rsrc, "ConfigurationPolicy", "bar",
  true, []string{name}, "reason", nil)...)
 
- sortRelatedObjectsAndUpdate(policy, relatedList, empty)
+ sortRelatedObjectsAndUpdate(policy, relatedList, empty, false)
  assert.True(t, relatedList[0].Object.Metadata.Namespace == "bar")
 
  // clear related objects and test sorting with no namespace
@@ -430,7 +430,7 @@ func TestSortRelatedObjectsAndUpdate(t *testing.T) {
  relatedList = append(relatedList, addRelatedObjects(true, rsrc, "ConfigurationPolicy", "",
  false, []string{name}, "reason", nil)...)
 
- sortRelatedObjectsAndUpdate(policy, relatedList, empty)
+ sortRelatedObjectsAndUpdate(policy, relatedList, empty, false)
  assert.True(t, relatedList[0].Object.Metadata.Name == "bar")
 }
 

controllers/metric.go

@@ -0,0 +1,116 @@
+package controllers
+
+import (
+ "fmt"
+ "sync"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "sigs.k8s.io/controller-runtime/pkg/metrics"
+
+ policyv1 "open-cluster-management.io/config-policy-controller/api/v1"
+)
+
+var (
+ evalLoopHistogram = prometheus.NewHistogram(
+ prometheus.HistogramOpts{
+ Name: "config_policies_evaluation_duration_seconds",
+ Help: "The seconds that it takes to evaluate all configuration policies on the cluster",
+ Buckets: []float64{1, 3, 9, 10.5, 15, 30, 60, 90, 120, 180, 300, 450, 600},
+ },
+ )
+ policyEvalSecondsCounter = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "config_policy_evaluation_seconds_total",
+ Help: "The total seconds taken while evaluating the configuration policy. Use this alongside " +
+ "config_policy_evaluation_total.",
+ },
+ []string{"name"},
+ )
+ policyEvalCounter = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "config_policy_evaluation_total",
+ Help: "The total number of evaluations of the configuration policy. Use this alongside " +
+ "config_policy_evaluation_seconds_total.",
+ },
+ []string{"name"},
+ )
+ compareObjSecondsCounter = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "compare_objects_seconds_total",
+ Help: "The total seconds taken while comparing policy objects. Use this alongside " +
+ "compare_objects_evaluation_total.",
+ },
+ []string{"config_policy_name", "namespace", "object"},
+ )
+ compareObjEvalCounter = prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "compare_objects_evaluation_total",
+ Help: "The total number of times the comparison algorithm is run on an object. " +
+ "Use this alongside compare_objects_seconds_total.",
+ },
+ []string{"config_policy_name", "namespace", "object"},
+ )
+ // The policyRelatedObjectMap collects a map of related objects to policies
+ // in order to populate the gauge:
+ // <kind.version/namespace/name>: []<policy-namespace/policy-name>
+ policyRelatedObjectMap sync.Map
+ policyRelatedObjectGauge = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Name: "common_related_objects",
+ Help: "A gauge vector of related objects managed by multiple policies.",
+ },
+ []string{
+ "relatedObject",
+ "policy",
+ },
+ )
+)
+
+func init() {
+ // Register custom metrics with the global Prometheus registry
+ metrics.Registry.MustRegister(evalLoopHistogram)
+ metrics.Registry.MustRegister(policyEvalSecondsCounter)
+ metrics.Registry.MustRegister(policyEvalCounter)
+ metrics.Registry.MustRegister(compareObjSecondsCounter)
+ metrics.Registry.MustRegister(compareObjEvalCounter)
+ metrics.Registry.MustRegister(policyRelatedObjectGauge)
+}
+
+// updateRelatedObjectMetric iterates through the collected related object map, deletes any metrics
+// that aren't duplications, and sets a metric for any related object that is handled by multiple
+// policies to the number of policies that currently handles it.
+func updateRelatedObjectMetric() {
+ log.V(3).Info("Updating common_related_objects metric ...")
+
+ policyRelatedObjectMap.Range(func(key any, value any) bool {
+ relatedObj := key.(string)
+ policies := value.([]string)
+
+ for _, policy := range policies {
+ if len(policies) == 1 {
+ policyRelatedObjectGauge.DeleteLabelValues(relatedObj, policy)
+
+ continue
+ }
+
+ gaugeInstance, err := policyRelatedObjectGauge.GetMetricWithLabelValues(relatedObj, policy)
+ if err != nil {
+ log.V(3).Error(err, "Failed to retrieve related object gauge")
+
+ continue
+ }
+
+ gaugeInstance.Set(float64(len(policies)))
+ }
+
+ return true
+ })
+}
+
+// getObjectString returns a string formatted as:
+// <kind>.<version>/<namespace>/<name>
+func getObjectString(obj policyv1.RelatedObject) string {
+ return fmt.Sprintf("%s.%s/%s/%s",
+ obj.Object.Kind, obj.Object.APIVersion,
+ obj.Object.Metadata.Namespace, obj.Object.Metadata.Name)
+}

main.go

@@ -72,7 +72,7 @@ func main() {
  var clusterName, hubConfigPath, targetKubeConfig, metricsAddr, probeAddr string
  var frequency uint
  var decryptionConcurrency, evaluationConcurrency uint8
- var enableLease, enableLeaderElection, legacyLeaderElection bool
+ var enableLease, enableLeaderElection, legacyLeaderElection, enableMetrics bool
 
  pflag.UintVar(&frequency, "update-frequency", 10,
  "The status update frequency (in seconds) of a mutation policy")
@@ -109,6 +109,7 @@ func main() {
  2,
  "The max number of concurrent configuration policy evaluations",
  )
+ pflag.BoolVar(&enableMetrics, "enable-metrics", true, "Disable custom metrics collection")
 
  pflag.Parse()
 
@@ -252,6 +253,7 @@ func main() {
  InstanceName: instanceName,
  TargetK8sClient: targetK8sClient,
  TargetK8sConfig: targetK8sConfig,
+ EnableMetrics: enableMetrics,
  }
  if err = reconciler.SetupWithManager(mgr); err != nil {
  log.Error(err, "Unable to create controller", "controller", "ConfigurationPolicy")