add some metrics; 1.node_cpu_cannot_be_reclaimed_seconds, 2.node_resource_recommended, 3.node_resource_recommended_from

pkg/metrics/ensuarance.go

@@ -26,6 +26,10 @@ const (
 	ExecutorErrorTotal    = "executor_error_total"
 	ExecutorEvictTotal    = "executor_evict_total"
 	PodResourceErrorTotal = "pod_resource_error_total"
+
+	NodeCpuCannotBeReclaimedSeconds = "node_cpu_cannot_be_reclaimed_seconds"
+	NodeResourceRecommended         = "node_resource_recommended"
+	NodeResourceRecommendedFrom     = "node_resource_recommended_from"
 )
 
 type StepLabel string
@@ -42,15 +46,18 @@ const (
 	// Step for pod resource manager
 	StepGetPeriod   StepLabel = "getPeriod"
 	StepUpdateQuota StepLabel = "updateQuota"
+
+	StepGetExtResourceRecommended StepLabel = "getExtResourceRecommended"
 )
 
 type SubComponent string
 
 const (
-	SubComponentSchedule    SubComponent = "schedule"
-	SubComponentThrottle    SubComponent = "throttle"
-	SubComponentEvict       SubComponent = "evict"
-	SubComponentPodResource SubComponent = "pod-resource-manager"
+	SubComponentSchedule     SubComponent = "schedule"
+	SubComponentThrottle     SubComponent = "throttle"
+	SubComponentEvict        SubComponent = "evict"
+	SubComponentPodResource  SubComponent = "pod-resource-manager"
+	SubComponentNodeResource SubComponent = "node-resource-manager"
 )
 
 type AnalyzeType string
@@ -180,6 +187,39 @@ var (
 			StabilityLevel: k8smetrics.ALPHA,
 		}, []string{"subcomponent", "step"},
 	)
+
+	// LastActivity records the last activity time of each steps
+	nodeCpuCannotBeReclaimedSeconds = k8smetrics.NewGaugeVec(
+		&k8smetrics.GaugeOpts{
+			Namespace:      CraneNamespace,
+			Subsystem:      CraneAgentSubsystem,
+			Name:           NodeCpuCannotBeReclaimedSeconds,
+			Help:           "The cpu seconds that cannot be reclaimed.",
+			StabilityLevel: k8smetrics.ALPHA,
+		}, []string{},
+	)
+
+	//NodeResourceRecommended
+	nodeResourceRecommended = k8smetrics.NewGaugeVec(
+		&k8smetrics.GaugeOpts{
+			Namespace:      CraneNamespace,
+			Subsystem:      CraneAgentSubsystem,
+			Name:           NodeResourceRecommended,
+			Help:           "The value of recommendation.",
+			StabilityLevel: k8smetrics.ALPHA,
+		}, []string{"subcomponent", "step", "resourceName"},
+	)
+
+	//NodeResourceRecommended
+	nodeResourceRecommendedFrom = k8smetrics.NewGaugeVec(
+		&k8smetrics.GaugeOpts{
+			Namespace:      CraneNamespace,
+			Subsystem:      CraneAgentSubsystem,
+			Name:           NodeResourceRecommendedFrom,
+			Help:           "Where the recommended values come from. (tsp: 1, local: 0)",
+			StabilityLevel: k8smetrics.ALPHA,
+		}, []string{"subcomponent", "step", "resourceName"},
+	)
 )
 
 var registerCraneAgentMetricsOnce sync.Once
@@ -195,6 +235,9 @@ func RegisterCraneAgent() {
 		legacyregistry.MustRegister(executorStatusCounts)
 		legacyregistry.MustRegister(executorErrorCounts)
 		legacyregistry.MustRegister(executorEvictCounts)
+		legacyregistry.MustRegister(nodeCpuCannotBeReclaimedSeconds)
+		legacyregistry.MustRegister(nodeResourceRecommended)
+		legacyregistry.MustRegister(nodeResourceRecommendedFrom)
 	})
 }
 
@@ -258,3 +301,23 @@ func PodResourceUpdateErrorCounterInc(subComponent SubComponent, stepName StepLa
 func ExecutorEvictCountsInc() {
 	executorEvictCounts.Inc()
 }
+
+func UpdateNodeCpuCannotBeReclaimedSeconds(value float64) {
+	nodeCpuCannotBeReclaimedSeconds.With(prometheus.Labels{}).Set(value)
+}
+
+func UpdateNodeResourceRecommendedValue(subComponent SubComponent, stepName StepLabel, resourceName string, from string, value float64) {
+	nodeResourceRecommended.With(prometheus.Labels{"subcomponent": string(subComponent), "step": string(stepName), "resourceName": resourceName}).Set(value)
+	switch from {
+	case "tsp":
+		UpdateNodeResourceRecommendedFromValue(subComponent, stepName, resourceName, 1)
+	case "local":
+		UpdateNodeResourceRecommendedFromValue(subComponent, stepName, resourceName, 0)
+	default:
+		UpdateNodeResourceRecommendedFromValue(subComponent, stepName, resourceName, -1)
+	}
+}
+
+func UpdateNodeResourceRecommendedFromValue(subComponent SubComponent, stepName StepLabel, resourceName string, value float64) {
+	nodeResourceRecommendedFrom.With(prometheus.Labels{"subcomponent": string(subComponent), "step": string(stepName), "resourceName": resourceName}).Set(value)
+}

pkg/resource/node_resource_manager.go

@@ -228,6 +228,7 @@ func (o *NodeResourceManager) BuildNodeStatus(node *v1.Node) map[v1.ResourceName
 		if nextRecommendation < 0 {
 			nextRecommendation = 0
 		}
+		metrics.UpdateNodeResourceRecommendedValue(metrics.SubComponentNodeResource, metrics.StepGetExtResourceRecommended, string(resourceName), resourceFrom, nextRecommendation)
 		extResourceName := fmt.Sprintf(utils.ExtResourcePrefixFormat, string(resourceName))
 		resValue, exists := node.Status.Capacity[v1.ResourceName(extResourceName)]
 		if exists && resValue.Value() != 0 &&
@@ -335,7 +336,9 @@ func (o *NodeResourceManager) GetCpuCoreCanNotBeReclaimedFromLocal() float64 {
 
 	// 1. Exclusive tethered CPU cannot be reclaimed even if the free part is free, so add the exclusive CPUIdle to the CanNotBeReclaimed CPU
 	// 2. The CPU used by extRes-container needs to be reclaimed, otherwise it will be double-counted due to the allotted mechanism of k8s, so the extResContainerCpuUsageTotal is subtracted from the CanNotBeReclaimedCpu
-	return nodeCpuUsageTotal + exclusiveCPUIdle - extResContainerCpuUsageTotal
+	nodeCpuCannotBeReclaimedSeconds := nodeCpuUsageTotal + exclusiveCPUIdle - extResContainerCpuUsageTotal
+	metrics.UpdateNodeCpuCannotBeReclaimedSeconds(nodeCpuCannotBeReclaimedSeconds)
+	return nodeCpuCannotBeReclaimedSeconds
 }
 
 func getReserveResourcePercentFromNodeAnnotations(annotations map[string]string, resourceName string) (float64, bool) {