feat: monitoring accuracy and latency of reported information in KCNR

cmd/katalyst-controller/app/controller/monitor.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+
+	"k8s.io/klog/v2"
+
+	katalystbase "github.com/kubewharf/katalyst-core/cmd/base"
+	"github.com/kubewharf/katalyst-core/pkg/config"
+	"github.com/kubewharf/katalyst-core/pkg/controller/monitor"
+)
+
+const (
+	MonitorControllerName = "monitor"
+)
+
+func StartMonitorController(ctx context.Context, controlCtx *katalystbase.GenericContext,
+	conf *config.Configuration, _ interface{}, _ string) (bool, error) {
+	var (
+		cnrMonitorController *monitor.CNRMonitorController
+		err                  error
+	)
+
+	if conf.MonitorConfig.EnableCNRMonitor {
+		cnrMonitorController, err = monitor.NewCNRMonitorController(ctx,
+			conf.GenericConfiguration,
+			conf.GenericControllerConfiguration,
+			conf.CNRMonitorConfig,
+			controlCtx.Client,
+			controlCtx.KubeInformerFactory.Core().V1().Nodes(),
+			controlCtx.KubeInformerFactory.Core().V1().Pods(),
+			controlCtx.InternalInformerFactory.Node().V1alpha1().CustomNodeResources(),
+			controlCtx.EmitterPool.GetDefaultMetricsEmitter(),
+		)
+		if err != nil {
+			klog.Errorf("failed to new CNR monitor controller")
+			return false, err
+		}
+	}
+
+	if cnrMonitorController != nil {
+		go cnrMonitorController.Run()
+	}
+
+	return true, nil
+}

cmd/katalyst-controller/app/enablecontrollers.go

@@ -51,6 +51,7 @@ func init() {
 	controllerInitializers.Store(controller.KCCControllerName, ControllerStarter{Starter: controller.StartKCCController})
 	controllerInitializers.Store(controller.SPDControllerName, ControllerStarter{Starter: controller.StartSPDController})
 	controllerInitializers.Store(controller.LifeCycleControllerName, ControllerStarter{Starter: controller.StartLifeCycleController})
+	controllerInitializers.Store(controller.MonitorControllerName, ControllerStarter{Starter: controller.StartMonitorController})
 }
 
 // RegisterControllerInitializer is used to register user-defined controllers

cmd/katalyst-controller/app/options/controller.go

@@ -28,6 +28,7 @@ type ControllersOptions struct {
 	*KCCOptions
 	*SPDOptions
 	*LifeCycleOptions
+	*MonitorOptions
 }
 
 func NewControllersOptions() *ControllersOptions {
@@ -36,6 +37,7 @@ func NewControllersOptions() *ControllersOptions {
 		KCCOptions:       NewKCCOptions(),
 		SPDOptions:       NewSPDOptions(),
 		LifeCycleOptions: NewLifeCycleOptions(),
+		MonitorOptions:   NewMonitorOptions(),
 	}
 }
 
@@ -44,6 +46,7 @@ func (o *ControllersOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 	o.KCCOptions.AddFlags(fss)
 	o.SPDOptions.AddFlags(fss)
 	o.LifeCycleOptions.AddFlags(fss)
+	o.MonitorOptions.AddFlags(fss)
 }
 
 // ApplyTo fills up config with options
@@ -54,6 +57,7 @@ func (o *ControllersOptions) ApplyTo(c *controllerconfig.ControllersConfiguratio
 	errList = append(errList, o.KCCOptions.ApplyTo(c.KCCConfig))
 	errList = append(errList, o.SPDOptions.ApplyTo(c.SPDConfig))
 	errList = append(errList, o.LifeCycleOptions.ApplyTo(c.LifeCycleConfig))
+	errList = append(errList, o.MonitorOptions.ApplyTo(c.MonitorConfig))
 	return errors.NewAggregate(errList)
 }
 

cmd/katalyst-controller/app/options/monitor.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	cliflag "k8s.io/component-base/cli/flag"
+
+	"github.com/kubewharf/katalyst-core/pkg/config/controller"
+)
+
+// MonitorOptions holds the configurations for Monitor.
+type MonitorOptions struct {
+	// EnableCNRMonitor is a flag to enable CNR monitor controller
+	EnableCNRMonitor bool
+}
+
+func NewMonitorOptions() *MonitorOptions {
+	return &MonitorOptions{
+		EnableCNRMonitor: true,
+	}
+}
+
+// AddFlags adds flags  to the specified FlagSet.
+func (o *MonitorOptions) AddFlags(fss *cliflag.NamedFlagSets) {
+	fs := fss.FlagSet("monitor")
+
+	fs.BoolVar(&o.EnableCNRMonitor, "cnr-monitor-enable", o.EnableCNRMonitor,
+		"whether to enable the cnr controller")
+}
+
+// ApplyTo fills up config with options
+func (o *MonitorOptions) ApplyTo(c *controller.MonitorConfig) error {
+	c.EnableCNRMonitor = o.EnableCNRMonitor
+	return nil
+}
+
+func (o *MonitorOptions) Config() (*controller.MonitorConfig, error) {
+	c := &controller.MonitorConfig{}
+	if err := o.ApplyTo(c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

examples/dedicated-normal-pod.yaml

@@ -0,0 +1,42 @@
+# Copyright 2022 The Katalyst Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  annotations:
+    "katalyst.kubewharf.io/qos_level": dedicated_cores
+    "katalyst.kubewharf.io/memory_enhancement": '{
+      "numa_binding": "true", 
+      "numa_exclusive": "true"
+    }'
+  name: dedicated-normal-pod
+  namespace: default
+spec:
+  containers:
+    - name: stress
+      image: joedval/stress:latest
+      command:
+        - stress
+        - -c
+        - "1"
+      imagePullPolicy: IfNotPresent
+      resources:
+        requests:
+          cpu: "1"
+          memory: 1Gi
+        limits:
+          cpu: "1"
+          memory: 1Gi
+  schedulerName: katalyst-scheduler

pkg/config/controller/controller_base.go

@@ -46,6 +46,7 @@ type ControllersConfiguration struct {
 	*KCCConfig
 	*SPDConfig
 	*LifeCycleConfig
+	*MonitorConfig
 }
 
 func NewGenericControllerConfiguration() *GenericControllerConfiguration {
@@ -58,5 +59,6 @@ func NewControllersConfiguration() *ControllersConfiguration {
 		KCCConfig:       NewKCCConfig(),
 		SPDConfig:       NewSPDConfig(),
 		LifeCycleConfig: NewLifeCycleConfig(),
+		MonitorConfig:   NewMonitorConfig(),
 	}
 }

pkg/config/controller/monitor.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+type CNRMonitorConfig struct{}
+
+type MonitorConfig struct {
+	// EnableCNRMonitor is a flag to enable CNR monitor controller
+	EnableCNRMonitor bool
+
+	*CNRMonitorConfig
+}
+
+func NewMonitorConfig() *MonitorConfig {
+	return &MonitorConfig{
+		EnableCNRMonitor: true,
+		CNRMonitorConfig: &CNRMonitorConfig{},
+	}
+}

pkg/controller/lifecycle/cnr.go

@@ -184,7 +184,7 @@ func (cl *CNRLifecycle) addCNREventHandle(obj interface{}) {
 func (cl *CNRLifecycle) updateCNREventHandle(_, new interface{}) {
 	c, ok := new.(*apis.CustomNodeResource)
 	if !ok {
-		klog.Errorf("cannot convert oldObj to *apis.CNR: %v", c)
+		klog.Errorf("cannot convert newObj to *apis.CNR: %v", c)
 		return
 	}
 	klog.V(4).Infof("notice addition of cnr %s", c.Name)