feat(qrm): introduce network QRM plugin and support packet tagging

cmd/katalyst-agent/app/agent/qrm/network_plugin.go

@@ -0,0 +1,58 @@
+// Copyright 2022 The Katalyst Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qrm
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/kubewharf/katalyst-core/cmd/katalyst-agent/app/agent"
+	"github.com/kubewharf/katalyst-core/pkg/config"
+)
+
+const (
+	QRMPluginNameNetwork = "qrm_network_plugin"
+)
+
+// networkPolicyInitializers is used to store the initializing function for network resource plugin policies
+var networkPolicyInitializers sync.Map
+
+// RegisterNetworkPolicyInitializer is used to register user-defined resource plugin init functions
+func RegisterNetworkPolicyInitializer(name string, initFunc agent.InitFunc) {
+	networkPolicyInitializers.Store(name, initFunc)
+}
+
+// getNetworkPolicyInitializers returns those policies with initialized functions
+func getNetworkPolicyInitializers() map[string]agent.InitFunc {
+	agents := make(map[string]agent.InitFunc)
+	networkPolicyInitializers.Range(func(key, value interface{}) bool {
+		agents[key.(string)] = value.(agent.InitFunc)
+		return true
+	})
+	return agents
+}
+
+// InitQRMNetworkPlugins initializes the network QRM plugins
+func InitQRMNetworkPlugins(agentCtx *agent.GenericContext, conf *config.Configuration, extraConf interface{}, agentName string) (bool, agent.Component, error) {
+	initializers := getNetworkPolicyInitializers()
+	policyName := conf.NetworkQRMPluginConfig.PolicyName
+
+	initFunc, ok := initializers[policyName]
+	if !ok {
+		return false, agent.ComponentStub{}, fmt.Errorf("invalid policy name %v for network resource plugin", policyName)
+	}
+
+	return initFunc(agentCtx, conf, extraConf, agentName)
+}

cmd/katalyst-agent/app/enableagents.go

@@ -47,6 +47,7 @@ func init() {
 	// qrm plugins are registered at top level of agent
 	agentInitializers.Store(qrm.QRMPluginNameCPU, AgentStarter{Init: qrm.InitQRMCPUPlugins})
 	agentInitializers.Store(qrm.QRMPluginNameMemory, AgentStarter{Init: qrm.InitQRMMemoryPlugins})
+	agentInitializers.Store(qrm.QRMPluginNameNetwork, AgentStarter{Init: qrm.InitQRMNetworkPlugins})
 }
 
 // RegisterAgentInitializer is used to register user-defined agents

cmd/katalyst-agent/app/options/qrm/network_plugin.go

@@ -0,0 +1,79 @@
+// Copyright 2022 The Katalyst Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qrm
+
+import (
+	cliflag "k8s.io/component-base/cli/flag"
+
+	"github.com/kubewharf/katalyst-api/pkg/consts"
+	qrmconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/qrm"
+)
+
+type NetworkOptions struct {
+	PolicyName                    string
+	NetClass                      NetClassOptions
+	PodLevelNetClassAnnoKey       string
+	PodLevelNetAttributesAnnoKeys string
+}
+
+type NetClassOptions struct {
+	// ReclaimedCores is the network class id for reclaimed_cores
+	ReclaimedCores uint32
+	// SharedCores is the network class id for shared_cores
+	SharedCores uint32
+	// DedicatedCores is the network class id for dedicated_cores
+	DedicatedCores uint32
+	// SystemCores is the network class id for system_cores
+	SystemCores uint32
+}
+
+func NewNetworkOptions() *NetworkOptions {
+	return &NetworkOptions{
+		PolicyName:                    "dynamic",
+		PodLevelNetClassAnnoKey:       consts.PodAnnotationNetClassKey,
+		PodLevelNetAttributesAnnoKeys: "",
+	}
+}
+
+func (o *NetworkOptions) AddFlags(fss *cliflag.NamedFlagSets) {
+	fs := fss.FlagSet("network_resource_plugin")
+
+	fs.StringVar(&o.PolicyName, "network-resource-plugin-policy",
+		o.PolicyName, "The policy network resource plugin should use")
+	fs.Uint32Var(&o.NetClass.ReclaimedCores, "network-resource-plugin-class-id-reclaimed-cores",
+		o.NetClass.ReclaimedCores, "net class id for reclaimed_cores")
+	fs.Uint32Var(&o.NetClass.SharedCores, "network-resource-plugin-class-id-shared-cores",
+		o.NetClass.SharedCores, "net class id for shared_cores")
+	fs.Uint32Var(&o.NetClass.DedicatedCores, "network-resource-plugin-class-id-dedicated-cores",
+		o.NetClass.DedicatedCores, "net class id for dedicated_cores")
+	fs.Uint32Var(&o.NetClass.SystemCores, "network-resource-plugin-class-id-system-cores",
+		o.NetClass.SystemCores, "net class id for system_cores")
+	fs.StringVar(&o.PodLevelNetClassAnnoKey, "network-resource-plugin-net-class-annotation-key",
+		o.PodLevelNetClassAnnoKey, "The annotation key of pod-level net class")
+	fs.StringVar(&o.PodLevelNetAttributesAnnoKeys, "network-resource-plugin-net-attributes-keys",
+		o.PodLevelNetAttributesAnnoKeys, "The annotation keys of pod-level network attributes, separated by commas")
+}
+
+func (o *NetworkOptions) ApplyTo(conf *qrmconfig.NetworkQRMPluginConfig) error {
+	conf.PolicyName = o.PolicyName
+	conf.NetClass.ReclaimedCores = o.NetClass.ReclaimedCores
+	conf.NetClass.SharedCores = o.NetClass.SharedCores
+	conf.NetClass.DedicatedCores = o.NetClass.DedicatedCores
+	conf.NetClass.SystemCores = o.NetClass.SystemCores
+	conf.PodLevelNetClassAnnoKey = o.PodLevelNetClassAnnoKey
+	conf.PodLevelNetAttributesAnnoKeys = o.PodLevelNetAttributesAnnoKeys
+
+	return nil
+}

cmd/katalyst-agent/app/options/qrm/qrm_base.go

@@ -58,20 +58,23 @@ func (o *GenericQRMPluginOptions) ApplyTo(conf *qrmconfig.GenericQRMPluginConfig
 }
 
 type QRMPluginsOptions struct {
-	CPUOptions    *CPUOptions
-	MemoryOptions *MemoryOptions
+	CPUOptions     *CPUOptions
+	MemoryOptions  *MemoryOptions
+	NetworkOptions *NetworkOptions
 }
 
 func NewQRMPluginsOptions() *QRMPluginsOptions {
 	return &QRMPluginsOptions{
-		CPUOptions:    NewCPUOptions(),
-		MemoryOptions: NewMemoryOptions(),
+		CPUOptions:     NewCPUOptions(),
+		MemoryOptions:  NewMemoryOptions(),
+		NetworkOptions: NewNetworkOptions(),
 	}
 }
 
 func (o *QRMPluginsOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 	o.CPUOptions.AddFlags(fss)
 	o.MemoryOptions.AddFlags(fss)
+	o.NetworkOptions.AddFlags(fss)
 }
 
 func (o *QRMPluginsOptions) ApplyTo(conf *qrmconfig.QRMPluginsConfiguration) error {
@@ -81,5 +84,8 @@ func (o *QRMPluginsOptions) ApplyTo(conf *qrmconfig.QRMPluginsConfiguration) err
 	if err := o.MemoryOptions.ApplyTo(conf.MemoryQRMPluginConfig); err != nil {
 		return err
 	}
+	if err := o.NetworkOptions.ApplyTo(conf.NetworkQRMPluginConfig); err != nil {
+		return err
+	}
 	return nil
 }

go.mod

@@ -140,7 +140,7 @@ require (
 )
 
 replace (
-	github.com/kubewharf/katalyst-api => github.com/kubewharf/katalyst-api v0.0.4
+	github.com/kubewharf/katalyst-api => github.com/kubewharf/katalyst-api v0.1.1
 	k8s.io/api => k8s.io/api v0.24.6
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.24.6
 	k8s.io/apimachinery => k8s.io/apimachinery v0.24.6

go.sum

@@ -550,8 +550,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kubewharf/katalyst-api v0.0.4 h1:P4ezSfAN65+fPIpoQiAcGy03EQ2gbL/R+WRZRwG7QeM=
-github.com/kubewharf/katalyst-api v0.0.4/go.mod h1:N477ZHDwzROGwkee/sPGs2mo5lWqtS10hZxYfTcxB88=
+github.com/kubewharf/katalyst-api v0.1.1 h1:Nik2aN3nmsOVKew60Wa/hLnJcc5JkGjSbgeBmEKxI1Y=
+github.com/kubewharf/katalyst-api v0.1.1/go.mod h1:N477ZHDwzROGwkee/sPGs2mo5lWqtS10hZxYfTcxB88=
 github.com/kubewharf/kubelet v1.24.6-kubewharf.4 h1:3lg/wqi0jqZZr60JcjDHeOpLcXMKsq3MrEq/4bmfzR8=
 github.com/kubewharf/kubelet v1.24.6-kubewharf.4/go.mod h1:MxbSZUx3wXztFneeelwWWlX7NAAStJ6expqq7gY2J3c=
 github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8=