Create extended resources with NodeFeatureRule

Signed-off-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
Signed-off-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com>

Co-authored-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com>
Co-authored-by: Fabiano Fidêncio <fabiano.fidencio@intel.com>
Signed-off-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com>

Dockerfile_generator

@@ -2,7 +2,7 @@ ARG BUILDER_IMAGE
 FROM ${BUILDER_IMAGE} as builder
 
 # Install tools
-RUN go install github.com/vektra/mockery/v2@v2.20.0 && \
+RUN go install github.com/vektra/mockery/v2@v2.21.6 && \
     go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2 && \
     git clone https://github.com/kubernetes/code-generator -b v0.24.3 --depth 1 && \
     go install k8s.io/code-generator/cmd/...@v0.24.3 && \

cmd/nfd-master/main.go

@@ -55,6 +55,8 @@ func main() {
 		switch f.Name {
 		case "featurerules-controller":
 			klog.Warningf("-featurerules-controller is deprecated, use '-crd-controller' flag instead")
+		case "resource-labels":
+			klog.Warningf("-resource-labels is deprecated, use a NodeFeatureRule CR instead")
 		}
 	})
 

deployment/base/nfd-crds/nfd-api-crds.yaml

@@ -36,6 +36,12 @@ spec:
           spec:
             description: NodeFeatureSpec describes a NodeFeature object.
             properties:
+              extendedResources:
+                additionalProperties:
+                  type: string
+                description: ExtendedResources is the set of extended resources that
+                  are requested to be created.
+                type: object
               features:
                 description: Features is the full "raw" features data that has been
                   discovered.
@@ -155,6 +161,11 @@ spec:
                   description: Rule defines a rule for node customization such as
                     labeling.
                   properties:
+                    extendedResources:
+                      description: ExtendedResources to create if the rule matches.
+                      items:
+                        type: string
+                      type: array
                     labels:
                       additionalProperties:
                         type: string

deployment/base/rbac/master-clusterrole.yaml

@@ -7,6 +7,7 @@ rules:
   - ""
   resources:
   - nodes
+  - nodes/status
   verbs:
   - get
   - patch

deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml

@@ -36,6 +36,12 @@ spec:
           spec:
             description: NodeFeatureSpec describes a NodeFeature object.
             properties:
+              extendedResources:
+                additionalProperties:
+                  type: string
+                description: ExtendedResources is the set of extended resources that
+                  are requested to be created.
+                type: object
               features:
                 description: Features is the full "raw" features data that has been
                   discovered.
@@ -155,6 +161,11 @@ spec:
                   description: Rule defines a rule for node customization such as
                     labeling.
                   properties:
+                    extendedResources:
+                      description: ExtendedResources to create if the rule matches.
+                      items:
+                        type: string
+                      type: array
                     labels:
                       additionalProperties:
                         type: string

deployment/helm/node-feature-discovery/templates/clusterrole.yaml

@@ -10,9 +10,7 @@ rules:
   - ""
   resources:
   - nodes
-{{- if .Values.master.resourceLabels | empty | not }}
   - nodes/status
-{{- end }}
   verbs:
   - get
   - patch

docs/reference/master-commandline-reference.md

@@ -256,6 +256,9 @@ nfd-master -deny-label-ns=*.vendor.com,vendor-2.io
 
 ### -resource-labels
 
+**DEPRECATED**: use a NodeFeatureRule CR instead [NodeFeatureRule](../usage/custom-resources.md#NodeFeatureRule)
+instead.
+
 The `-resource-labels` flag specifies a comma-separated list of features to be
 advertised as extended resources instead of labels. Features that have integer
 values can be published as Extended Resources by listing them in this flag.

docs/usage/customization-guide.md

@@ -475,7 +475,7 @@ details.
 labels specified in the `labels` field will override anything
 originating from `labelsTemplate`.
 
-### Taints
+#### Taints
 
 *taints* is a list of taint entries and each entry can have `key`, `value` and `effect`,
 where the `value` is optional. Effect could be `NoSchedule`, `PreferNoSchedule`
@@ -491,6 +491,42 @@ rules to use. In other words, these are variables that are not advertised as
 node labels. See [backreferences](#backreferences) for more details on the
 usage of vars.
 
+#### Extended resources
+
+The `.extendedResources` field is a list of extended resources to advertise.
+See [extended resources](#extended-resources) for more details.
+
+Take this rule as a referential example:
+
+```yaml
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeatureRule
+metadata:
+  name: tdx-total-keys
+spec:
+  rules:
+    - name: "TDX total keys rule"
+      extendedResources:
+        - "cpu-security.tdx.total_keys"
+      matchFeatures:
+        - feature: cpu.security
+          matchExpressions:
+            tdx.enabled: {op: Exists}
+```
+
+This will yield into the following node status:
+
+```yaml
+  allocatable:
+    ...
+    feature.node.kubernetes.io/cpu-security.tdx.total_keys: "31"
+    ...
+  capacity:
+    ...
+    feature.node.kubernetes.io/cpu-security.tdx.total_keys: "31"
+    ...
+```
+
 #### Vars template
 
 The `.varsTemplate` field specifies a text template for dynamically creating

pkg/apis/nfd/v1alpha1/annotations_labels.go

@@ -32,6 +32,9 @@ const (
 	// AnnotationNs namespace for all NFD-related annotations.
 	AnnotationNs = "nfd.node.kubernetes.io"
 
+	// ExtendedResourceNs is the namespace for extended resources.
+	ExtendedResourceNs = "nfd.node.kubernetes.io"
+
 	// ExtendedResourceAnnotation is the annotation that holds all extended resources managed by NFD.
 	ExtendedResourceAnnotation = AnnotationNs + "/extended-resources"
 

pkg/apis/nfd/v1alpha1/rule.go

@@ -30,13 +30,15 @@ import (
 // RuleOutput contains the output out rule execution.
 // +k8s:deepcopy-gen=false
 type RuleOutput struct {
-	Labels map[string]string
-	Vars   map[string]string
-	Taints []corev1.Taint
+	ExtendedResources []string
+	Labels            map[string]string
+	Vars              map[string]string
+	Taints            []corev1.Taint
 }
 
 // Execute the rule against a set of input features.
 func (r *Rule) Execute(features *Features) (RuleOutput, error) {
+	extendedResources := []string{}
 	labels := make(map[string]string)
 	vars := make(map[string]string)
 
@@ -88,14 +90,16 @@ func (r *Rule) Execute(features *Features) (RuleOutput, error) {
 		}
 	}
 
+	extendedResources = append(extendedResources, r.ExtendedResources...)
+
 	for k, v := range r.Labels {
 		labels[k] = v
 	}
 	for k, v := range r.Vars {
 		vars[k] = v
 	}
 
-	ret := RuleOutput{Labels: labels, Vars: vars, Taints: r.Taints}
+	ret := RuleOutput{ExtendedResources: extendedResources, Labels: labels, Vars: vars, Taints: r.Taints}
 	utils.KlogDump(2, fmt.Sprintf("rule %q matched with: ", r.Name), "  ", ret)
 	return ret, nil
 }

pkg/apis/nfd/v1alpha1/types.go

@@ -51,6 +51,10 @@ type NodeFeatureSpec struct {
 	// Labels is the set of node labels that are requested to be created.
 	// +optional
 	Labels map[string]string `json:"labels"`
+	// ExtendedResources is the set of extended resources that are requested to
+	// be created.
+	// +optional
+	ExtendedResources map[string]string `json:"extendedResources"`
 }
 
 // Features is the collection of all discovered features.
@@ -163,6 +167,10 @@ type Rule struct {
 	// +optional
 	Taints []corev1.Taint `json:"taints,omitempty"`
 
+	// ExtendedResources to create if the rule matches.
+	// +optional
+	ExtendedResources []string `json:"extendedResources"`
+
 	// MatchFeatures specifies a set of matcher terms all of which must match.
 	// +optional
 	MatchFeatures FeatureMatcher `json:"matchFeatures"`

pkg/nfd-master/nfd-master.go

@@ -35,8 +35,7 @@ import (
 	"google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/peer"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/labels"
-	label "k8s.io/apimachinery/pkg/labels"
+	k8sLabels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
@@ -409,8 +408,9 @@ func (m *nfdMaster) updateMasterNode() error {
 // into extended resources. This function also handles proper namespacing of
 // labels and ERs, i.e. adds the possibly missing default namespace for labels
 // arriving through the gRPC API.
-func (m *nfdMaster) filterFeatureLabels(labels Labels) (Labels, ExtendedResources) {
+func (m *nfdMaster) filterFeatureLabels(labels Labels, extendedResources []string) (Labels, ExtendedResources) {
 	outLabels := Labels{}
+	outExtendedResources := ExtendedResources{}
 
 	for label, value := range labels {
 		// Add possibly missing default ns
@@ -438,8 +438,8 @@ func (m *nfdMaster) filterFeatureLabels(labels Labels) (Labels, ExtendedResource
 		outLabels[label] = value
 	}
 
-	// Remove labels which are intended to be extended resources
-	extendedResources := ExtendedResources{}
+	// via -resource-labels
+	// TODO: Deprecate -resource-labels in favor of NodeFeatureRules CRD
 	for extendedResourceName := range m.args.ResourceLabels {
 		// Add possibly missing default ns
 		extendedResourceName = addNs(extendedResourceName, nfdv1alpha1.FeatureLabelNs)
@@ -449,12 +449,27 @@ func (m *nfdMaster) filterFeatureLabels(labels Labels) (Labels, ExtendedResource
 				continue // non-numeric label can't be used
 			}
 
-			extendedResources[extendedResourceName] = value
+			outExtendedResources[extendedResourceName] = value
 			delete(outLabels, extendedResourceName)
 		}
 	}
 
-	return outLabels, extendedResources
+	// via NodeFeatureRules CR
+	for _, extendedResourceName := range extendedResources {
+		// Add possibly missing default ns
+		extendedResourceName = addNs(extendedResourceName, nfdv1alpha1.FeatureLabelNs)
+		if value, ok := outLabels[extendedResourceName]; ok {
+			if _, err := strconv.Atoi(value); err != nil {
+				klog.Errorf("bad label value (%s: %s) encountered for extended resource: %s", extendedResourceName, value, err.Error())
+				continue // non-numeric label can't be used
+			}
+
+			outExtendedResources[extendedResourceName] = value
+			delete(outLabels, extendedResourceName)
+		}
+	}
+
+	return outLabels, outExtendedResources
 }
 
 func verifyNodeName(cert *x509.Certificate, nodeName string) error {
@@ -539,7 +554,7 @@ func (m *nfdMaster) nfdAPIUpdateAllNodes() error {
 }
 
 func (m *nfdMaster) nfdAPIUpdateOneNode(nodeName string) error {
-	sel := labels.SelectorFromSet(labels.Set{nfdv1alpha1.NodeFeatureObjNodeNameLabel: nodeName})
+	sel := k8sLabels.SelectorFromSet(k8sLabels.Set{nfdv1alpha1.NodeFeatureObjNodeNameLabel: nodeName})
 	objs, err := m.nfdController.featureLister.List(sel)
 	if err != nil {
 		return fmt.Errorf("failed to get NodeFeature resources for node %q: %w", nodeName, err)
@@ -605,19 +620,22 @@ func (m *nfdMaster) nfdAPIUpdateOneNode(nodeName string) error {
 	return nil
 }
 
-func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName string, annotations, labels map[string]string, features *nfdv1alpha1.Features) error {
+func (m *nfdMaster) refreshNodeFeatures(cli *kubernetes.Clientset, nodeName string, annotations Annotations, labels map[string]string, features *nfdv1alpha1.Features) error {
+
 	if labels == nil {
 		labels = make(map[string]string)
 	}
 
-	crLabels, crTaints := m.processNodeFeatureRule(features)
+	crExtendedResources, crLabels, crTaints := m.processNodeFeatureRule(features)
 
 	// Mix in CR-originated labels
 	for k, v := range crLabels {
 		labels[k] = v
 	}
 
-	labels, extendedResources := m.filterFeatureLabels(labels)
+	// Remove labels which are intended to be extended resources or their NS
+	// is not whitelisted
+	labels, extendedResources := m.filterFeatureLabels(labels, crExtendedResources)
 
 	var taints []corev1.Taint
 	if m.args.EnableTaints {
@@ -738,21 +756,22 @@ func authorizeClient(c context.Context, checkNodeName bool, nodeName string) err
 	return nil
 }
 
-func (m *nfdMaster) processNodeFeatureRule(features *nfdv1alpha1.Features) (map[string]string, []corev1.Taint) {
+func (m *nfdMaster) processNodeFeatureRule(features *nfdv1alpha1.Features) ([]string, map[string]string, []corev1.Taint) {
 	if m.nfdController == nil {
-		return nil, nil
+		return nil, nil, nil
 	}
 
+	extendedResources := []string{}
 	labels := make(map[string]string)
 	var taints []corev1.Taint
-	ruleSpecs, err := m.nfdController.ruleLister.List(label.Everything())
+	ruleSpecs, err := m.nfdController.ruleLister.List(k8sLabels.Everything())
 	sort.Slice(ruleSpecs, func(i, j int) bool {
 		return ruleSpecs[i].Name < ruleSpecs[j].Name
 	})
 
 	if err != nil {
 		klog.Errorf("failed to list NodeFeatureRule resources: %v", err)
-		return nil, nil
+		return nil, nil, nil
 	}
 
 	// Process all rule CRs
@@ -775,13 +794,15 @@ func (m *nfdMaster) processNodeFeatureRule(features *nfdv1alpha1.Features) (map[
 				labels[k] = v
 			}
 
+			extendedResources = append(extendedResources, ruleOut.ExtendedResources...)
+
 			// Feed back rule output to features map for subsequent rules to match
 			features.InsertAttributeFeatures(nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Labels)
 			features.InsertAttributeFeatures(nfdv1alpha1.RuleBackrefDomain, nfdv1alpha1.RuleBackrefFeature, ruleOut.Vars)
 		}
 	}
 
-	return labels, taints
+	return extendedResources, labels, taints
 }
 
 // updateNodeObject ensures the Kubernetes node object is up to date,