Get Topology Manager policy from configz-endpoint

This patch allows nfd-master to pull the config from the Kubernetes configz endpoint. This update introduces --obtain-kubelet-config=<option> as it provides the option to obtain Kubelet config from configz-endpoint in addition to the already existing way of obtaining kubelet-config-file. Co-authored-by: Killian Muldoon <killian.muldoon@intel.com> Signed-off-by: Killian Muldoon <killian.muldoon@intel.com> Signed-off-by: Talor Itzhak <titzhak@redhat.com>
kubernetes-sigs · May 18, 2021 · 9c272c8 · 9c272c8
1 parent 0be8192
commit 9c272c8
Show file tree

Hide file tree

Showing 15 changed files with 172 additions and 47 deletions.
diff --git a/cmd/nfd-topology-updater/main.go b/cmd/nfd-topology-updater/main.go
@@ -36,6 +36,12 @@ const (
 	ProgramName = "nfd-topology-updater"
 )
 
+// Options to obtain Kubelet config file
+const (
+	KubeletFile = "kubelet-config-file"
+	Configz     = "configz-endpoint"
+)
+
 func main() {
 	// Assert that the version is known
 	if version.Undefined() {
@@ -47,12 +53,15 @@ func main() {
 		log.Fatalf("failed to parse command line: %v", err)
 	}
 
-	klConfig, err := kubeconf.GetKubeletConfigFromLocalFile(resourcemonitorArgs.KubeletConfigFile)
-	if err != nil {
-		log.Fatalf("error getting topology Manager Policy: %v", err)
-	}
-	tmPolicy := klConfig.TopologyManagerPolicy
-	log.Printf("Detected kubelet Topology Manager policy %q", tmPolicy)
+	var tmPolicy string
+	if resourcemonitorArgs.KubeletConfigObtainOpt == KubeletFile {
+		klConfig, err := kubeconf.GetKubeletConfigFromLocalFile(resourcemonitorArgs.KubeletConfigFile)
+		if err != nil {
+			log.Fatalf("error getting topology Manager Policy: %v", err)
+		}
+		tmPolicy = klConfig.TopologyManagerPolicy
+		log.Printf("Detected kubelet Topology Manager policy %q", tmPolicy)
+	} // Otherwise get TopologyManagerPolicy from configz-endpoint
 
 	podResClient, err := podres.GetPodResClient(resourcemonitorArgs.PodResourceSocketPath)
 	if err != nil {
@@ -125,38 +134,41 @@ func argsParse(argv []string) (topology.Args, resourcemonitor.Args, error) {
   %s [--no-publish] [--oneshot | --sleep-interval=<seconds>] [--server=<server>]
 	   [--server-name-override=<name>] [--ca-file=<path>] [--cert-file=<path>]
 		 [--key-file=<path>] [--container-runtime=<runtime>] [--podresources-socket=<path>]
-		 [--watch-namespace=<namespace>] [--sysfs=<mountpoint>] [--kubelet-config-file=<path>]
+		 [--watch-namespace=<namespace>] [--sysfs=<mountpoint>] [--obtain-kubelet-config=<option>] [--kubelet-config-file=<path>]
 
   %s -h | --help
   %s --version
 
   Options:
-  -h --help                       Show this screen.
-  --version                       Output version and exit.
-  --ca-file=<path>                Root certificate for verifying connections
-                                  [Default: ]
-  --cert-file=<path>              Certificate used for authenticating connections
-                                  [Default: ]
-  --key-file=<path>               Private key matching --cert-file
-                                  [Default: ]
-  --server=<server>               NFD server address to connect to.
-                                  [Default: localhost:8080]
-  --server-name-override=<name>   Name (CN) expect from server certificate, useful
-                                  in testing
-                                  [Default: ]
-  --no-publish                    Do not publish discovered features to the
-                                  cluster-local Kubernetes API server.
-  --oneshot                       Update once and exit.
-  --sleep-interval=<seconds>      Time to sleep between re-labeling. Non-positive
-                                  value implies no re-labeling (i.e. infinite
-                                  sleep). [Default: 60s]
-  --watch-namespace=<namespace>   Namespace to watch pods for. Use "" for all namespaces.
-  --sysfs=<mountpoint>            Mount point of the sysfs.
-                                  [Default: /host]
-  --kubelet-config-file=<path>    Kubelet config file path.
-                                  [Default: /podresources/config.yaml]
-  --podresources-socket=<path>    Pod Resource Socket path to use.
-                                  [Default: /podresources/kubelet.sock] `,
+  -h --help                        Show this screen.
+  --version                        Output version and exit.
+  --ca-file=<path>                 Root certificate for verifying connections
+                                   [Default: ]
+  --cert-file=<path>               Certificate used for authenticating connections
+                                   [Default: ]
+  --key-file=<path>                Private key matching --cert-file
+                                   [Default: ]
+  --server=<server>                NFD server address to connect to.
+                                   [Default: localhost:8080]
+  --server-name-override=<name>    Name (CN) expect from server certificate, useful
+                                   in testing
+                                   [Default: ]
+  --no-publish                     Do not publish discovered features to the
+                                   cluster-local Kubernetes API server.
+  --oneshot                        Update once and exit.
+  --sleep-interval=<seconds>       Time to sleep between re-labeling. Non-positive
+                                   value implies no re-labeling (i.e. infinite
+                                   sleep). [Default: 60s]
+  --watch-namespace=<namespace>    Namespace to watch pods for. Use "" for all namespaces.
+  --sysfs=<mountpoint>             Mount point of the sysfs.
+								   [Default: /host]
+  --obtain-kubelet-config=<option> Options to obtain Kubelet config file
+								   [Possible arguments: kubelet-config-file/configz-endpoint]
+								   [Default: kubelet-config-file]
+  --kubelet-config-file=<path>     Kubelet config file path.
+                                   [Default: /podresources/config.yaml]
+  --podresources-socket=<path>     Pod Resource Socket path to use.
+                                   [Default: /podresources/kubelet.sock] `,
 
 		ProgramName,
 		ProgramName,
@@ -183,8 +195,13 @@ func argsParse(argv []string) (topology.Args, resourcemonitor.Args, error) {
 	if ns, ok := arguments["--watch-namespace"].(string); ok {
 		resourcemonitorArgs.Namespace = ns
 	}
-	if kubeletConfigPath, ok := arguments["--kubelet-config-file"].(string); ok {
-		resourcemonitorArgs.KubeletConfigFile = kubeletConfigPath
+	if obtainOpt, ok := arguments["--obtain-kubelet-config"].(string); ok {
+		resourcemonitorArgs.KubeletConfigObtainOpt = obtainOpt
+	}
+	if resourcemonitorArgs.KubeletConfigObtainOpt == KubeletFile {
+		if kubeletConfigPath, ok := arguments["--kubelet-config-file"].(string); ok {
+			resourcemonitorArgs.KubeletConfigFile = kubeletConfigPath
+		}
 	}
 	resourcemonitorArgs.SysfsRoot = arguments["--sysfs"].(string)
 	if path, ok := arguments["--podresources-socket"].(string); ok {

diff --git a/cmd/nfd-topology-updater/main_test.go b/cmd/nfd-topology-updater/main_test.go
@@ -33,6 +33,7 @@ func TestArgsParse(t *testing.T) {
 				So(args.Oneshot, ShouldBeTrue)
 				So(finderArgs.SleepInterval, ShouldEqual, 60*time.Second)
 				So(finderArgs.SysfsRoot, ShouldEqual, "/host")
+				So(finderArgs.KubeletConfigObtainOpt, ShouldEqual, KubeletFile)
 				So(finderArgs.KubeletConfigFile, ShouldEqual, "/podresources/config.yaml")
 				So(finderArgs.PodResourceSocketPath, ShouldEqual, "/podresources/kubelet.sock")
 				So(err, ShouldBeNil)
@@ -61,6 +62,7 @@ func TestArgsParse(t *testing.T) {
 				So(args.Oneshot, ShouldBeFalse)
 				So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second)
 				So(finderArgs.SysfsRoot, ShouldEqual, "/host")
+				So(finderArgs.KubeletConfigObtainOpt, ShouldEqual, KubeletFile)
 				So(finderArgs.KubeletConfigFile, ShouldEqual, "/podresources/config.yaml")
 				So(finderArgs.PodResourceSocketPath, ShouldEqual, "/path/testkubelet.sock")
 				So(err, ShouldBeNil)
@@ -74,6 +76,7 @@ func TestArgsParse(t *testing.T) {
 				So(args.Oneshot, ShouldBeFalse)
 				So(finderArgs.SleepInterval, ShouldEqual, 30*time.Second)
 				So(finderArgs.SysfsRoot, ShouldEqual, "/host/sysfs-root")
+				So(finderArgs.KubeletConfigObtainOpt, ShouldEqual, KubeletFile)
 				So(finderArgs.KubeletConfigFile, ShouldEqual, "/podresources/config.yaml")
 				So(finderArgs.PodResourceSocketPath, ShouldEqual, "/podresources/kubelet.sock")
 				So(err, ShouldBeNil)

diff --git a/go.mod b/go.mod
@@ -8,12 +8,12 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/golang/protobuf v1.4.3
 	github.com/google/go-cmp v0.5.2
-	github.com/gorilla/context v1.1.1 // indirect
 	github.com/jaypipes/ghw v0.8.0
 	github.com/k8stopologyawareschedwg/noderesourcetopology-api v0.0.8
 	github.com/klauspost/cpuid/v2 v2.0.6
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.7.0
+	github.com/pkg/errors v0.9.1
 	github.com/smartystreets/assertions v1.2.0
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/stretchr/testify v1.6.1

diff --git a/go.sum b/go.sum
@@ -362,8 +362,6 @@ github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
-github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=

diff --git a/manifests/nfd-daemonset-combined.yaml.template b/manifests/nfd-daemonset-combined.yaml.template
@@ -134,6 +134,7 @@ spec:
           command:
             - "nfd-topology-updater"
           args:
+            - "--obtain-kubelet-config=kubelet-config-file"
             - "--kubelet-config-file=/podresources/config.yaml"
             - "--podresources-socket=/podresources/kubelet.sock"
             - "--sleep-interval=3s"

diff --git a/manifests/nfd-daemonset-master-topology-updater.yaml.template b/manifests/nfd-daemonset-master-topology-updater.yaml.template
@@ -21,6 +21,7 @@ rules:
       - ""
     resources:
       - nodes
+      - nodes/proxy
     # when using command line flag --resource-labels to create extended resources
     # you will need to uncomment "- nodes/status"
     # - nodes/status
@@ -103,6 +104,7 @@ spec:
           command:
             - "nfd-topology-updater"
           args:
+            - "--obtain-kubelet-config=kubelet-config-file"
             - "--kubelet-config-file=/podresources/config.yaml"
             - "--podresources-socket=/podresources/kubelet.sock"
             - "--sleep-interval=3s"

diff --git a/manifests/nfd-master.yaml.template b/manifests/nfd-master.yaml.template
@@ -20,6 +20,7 @@ rules:
   - ""
   resources:
   - nodes
+  - nodes/proxy
 # when using command line flag --resource-labels to create extended resources
 # you will need to uncomment "- nodes/status"
 # - nodes/status

diff --git a/manifests/nfd-topology-updater-daemonset.yaml.template b/manifests/nfd-topology-updater-daemonset.yaml.template
@@ -32,6 +32,7 @@ spec:
           command:
             - "nfd-topology-updater"
           args:
+            - "--obtain-kubelet-config=kubelet-config-file"
             - "--kubelet-config-file=/podresources/config.yaml"
             - "--podresources-socket=/podresources/kubelet.sock"
             - "--sleep-interval=3s"

diff --git a/manifests/nfd-topology-updater-job.yaml.template b/manifests/nfd-topology-updater-job.yaml.template
@@ -41,6 +41,7 @@ spec:
           command:
             - "nfd-topology-updater"
           args:
+            - "--obtain-kubelet-config=kubelet-config-file"
             - "--kubelet-config-file=/podresources/config.yaml"
             - "--podresources-socket=/podresources/kubelet.sock"
             - "--oneshot"

diff --git a/pkg/apihelper/apihelpers.go b/pkg/apihelper/apihelpers.go
@@ -20,6 +20,7 @@ import (
 	topologyclientset "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/clientset/versioned"
 	api "k8s.io/api/core/v1"
 	k8sclient "k8s.io/client-go/kubernetes"
+	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
 )
 
 // APIHelpers represents a set of API helpers for Kubernetes
@@ -44,4 +45,7 @@ type APIHelpers interface {
 
 	// GetTopologyClient returns a topologyclientset
 	GetTopologyClient() (*topologyclientset.Clientset, error)
+
+	// GetKubeletConfigs returns node's kubelet config file
+	GetKubeletConfig(c *k8sclient.Clientset, nodeName string) (*kubeletconfig.KubeletConfiguration, error)
 }
diff --git a/pkg/apihelper/k8shelpers.go b/pkg/apihelper/k8shelpers.go
@@ -21,12 +21,18 @@ import (
 	"encoding/json"
 
 	topologyclientset "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/generated/clientset/versioned"
+	"github.com/pkg/errors"
+
 	api "k8s.io/api/core/v1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	k8sclient "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
+	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
+	kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme"
 )
 
 // Implements APIHelpers
@@ -122,3 +128,42 @@ func (h K8sHelpers) PatchNodeStatus(c *k8sclient.Clientset, nodeName string, pat
 	return nil
 
 }
+
+func (h K8sHelpers) GetKubeletConfig(c *k8sclient.Clientset, nodeName string) (*kubeletconfig.KubeletConfiguration, error) {
+	//request here is the same as: curl https://$APISERVER:6443/api/v1/nodes/$NODE_NAME/proxy/configz
+	res := c.CoreV1().RESTClient().Get().Resource("nodes").Name(nodeName).SubResource("proxy").Suffix("configz").Do(context.TODO())
+	return decodeConfigz(&res)
+}
+
+// Decodes the rest result from /configz and returns a kubeletconfig.KubeletConfiguration (internal type).
+func decodeConfigz(res *rest.Result) (*kubeletconfig.KubeletConfiguration, error) {
+	// This hack because /configz reports the following structure:
+	// {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}}
+	type configzWrapper struct {
+		ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"`
+	}
+
+	configz := configzWrapper{}
+	kubeCfg := kubeletconfig.KubeletConfiguration{}
+
+	contentsBytes, err := res.Raw()
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(contentsBytes, &configz)
+	if err != nil {
+		return nil, errors.Wrap(err, string(contentsBytes))
+	}
+
+	scheme, _, err := kubeletconfigscheme.NewSchemeAndCodecs()
+	if err != nil {
+		return nil, err
+	}
+	err = scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return &kubeCfg, nil
+}
diff --git a/pkg/nfd-master/nfd-master.go b/pkg/nfd-master/nfd-master.go
@@ -462,6 +462,28 @@ func (m *nfdMaster) UpdateNodeTopology(c context.Context, r *topologypb.NodeTopo
 			return &topologypb.NodeTopologyResponse{}, fmt.Errorf("request authorization failed: cert valid for '%s', requested node name '%s'", cn, r.NodeName)
 		}
 	}
+
+	// We expect the data to be found on r.TopologyPolicies[0] or not to be found at all
+	for idx, policy := range r.TopologyPolicies {
+		if len(policy) != 0 && idx != 0 {
+			return &topologypb.NodeTopologyResponse{}, fmt.Errorf("Topology Policy error: policy %v not expected to be found at index %v", policy, idx)
+		}
+	}
+	if len(r.TopologyPolicies[0]) == 0 {
+		stdoutLogger.Printf("Warning: Using configz-endpoint in order to get Kubelet configuration, consider to be unstable")
+		cli, err := m.apihelper.GetClient()
+		if err != nil {
+			stderrLogger.Printf("%s", err.Error())
+			return &topologypb.NodeTopologyResponse{}, err
+		}
+		kc, err := m.apihelper.GetKubeletConfig(cli, r.NodeName)
+		if err != nil {
+			stderrLogger.Printf("failed to get Kubelet config: %s", err.Error())
+			return &topologypb.NodeTopologyResponse{}, err
+		}
+		r.TopologyPolicies[0] = kc.TopologyManagerPolicy
+	}
+
 	stdoutLogger.Printf("REQUEST Node: %s NFD-version: %s Topology Policy: %s Zones: %v", r.NodeName, r.NfdVersion, r.TopologyPolicies, dumpobject.DumpObject(r.Zones))
 
 	if !m.args.NoPublish {

diff --git a/pkg/resourcemonitor/types.go b/pkg/resourcemonitor/types.go
@@ -25,11 +25,12 @@ import (
 )
 
 type Args struct {
-	PodResourceSocketPath string
-	SleepInterval         time.Duration
-	Namespace             string
-	SysfsRoot             string
-	KubeletConfigFile     string
+	PodResourceSocketPath  string
+	SleepInterval          time.Duration
+	Namespace              string
+	SysfsRoot              string
+	KubeletConfigObtainOpt string
+	KubeletConfigFile      string
 }
 
 type ResourceInfo struct {