[WIP] PoC ResourceClass

cmd/kube-apiserver/app/server.go

@@ -689,6 +689,10 @@ func postProcessOpenAPISpecForBackwardCompatibility(s *spec.Swagger) (*spec.Swag
 		"v1beta1.ClusterRoleList":             "k8s.io/kubernetes/pkg/apis/rbac/v1beta1.ClusterRoleList",
 		"v1beta1.ResourceAttributes":          "k8s.io/kubernetes/pkg/apis/authorization/v1beta1.ResourceAttributes",
 		"v1.Pod":                              "k8s.io/kubernetes/pkg/api/v1.Pod",
+		"v1.ResourceClass":                    "k8s.io/kubernetes/pkg/api/v1.ResourceClass",
+		"v1.ResourceClassList":                "k8s.io/kubernetes/pkg/api/v1.ResourceClassList",
+		"v1.ResourceClassSpec":                "k8s.io/kubernetes/pkg/api/v1.ResourceClassSpec",
+		"v1.ResourceClassStatus":              "k8s.io/kubernetes/pkg/api/v1.ResourceClassStatus",
 		"v1.FCVolumeSource":                   "k8s.io/kubernetes/pkg/api/v1.FCVolumeSource",
 		"v1beta1.SubresourceReference":        "k8s.io/kubernetes/pkg/apis/extensions/v1beta1.SubresourceReference",
 		"v1.ResourceQuotaStatus":              "k8s.io/kubernetes/pkg/api/v1.ResourceQuotaStatus",

cmd/kubelet/app/options/options.go

@@ -150,6 +150,7 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&c.ExperimentalFailSwapOn, "experimental-fail-swap-on", c.ExperimentalFailSwapOn, "Makes the Kubelet fail to start if swap is enabled on the node. This is a temporary opton to maintain legacy behavior, failing due to swap enabled will happen by default in v1.6.")
 
 	fs.StringVar(&c.PodManifestPath, "pod-manifest-path", c.PodManifestPath, "Path to to the directory containing pod manifest files to run, or the path to a single pod manifest file. Files starting with dots will be ignored.")
+	fs.StringVar(&c.DeviceFilePath, "experimental-device-file-path", c.DeviceFilePath, "Path to the directory containing device yaml files to run, or the path to a single device file. Files starting with dots will be ignored.")
 	fs.DurationVar(&c.SyncFrequency.Duration, "sync-frequency", c.SyncFrequency.Duration, "Max period between synchronizing running containers and config")
 	fs.DurationVar(&c.FileCheckFrequency.Duration, "file-check-frequency", c.FileCheckFrequency.Duration, "Duration between checking config files for new data")
 	fs.DurationVar(&c.HTTPCheckFrequency.Duration, "http-check-frequency", c.HTTPCheckFrequency.Duration, "Duration between checking http for new data")

examples/resource-class/nvidia.tesla.gpu.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Device
+metadata:
+  name: nvidia-tesla-gpu
+  labels:
+    type: nvidia-gpu
+    quantity: "8"
+    compute-ability: "3.7"
+    memory: 10Gi
+    ecc: "true"
+    family: tesla
+    nvlink: "true"
+    model: k80
+

examples/resource-class/rclass.yml

@@ -0,0 +1,14 @@
+--- 
+kind: ResourceClass
+metadata: 
+  name: my.rc2
+spec: 
+  resourceSelector: 
+    - 
+      matchExpressions: 
+        - 
+          key: "Type"
+          operator: "In"
+          values: 
+            - "nvidia-gpu"
+

pkg/api/install/install.go

@@ -44,6 +44,7 @@ func Install(groupFactoryRegistry announced.APIGroupFactoryRegistry, registry *r
 				"Namespace",
 				"PersistentVolume",
 				"ComponentStatus",
+				"ResourceClass",
 			),
 			IgnoredKinds: sets.NewString(
 				"ListOptions",

pkg/api/register.go

@@ -94,6 +94,9 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&List{},
 		&LimitRange{},
 		&LimitRangeList{},
+		&ResourceClass{},
+		&ResourceClassList{},
+		&Device{},
 		&ResourceQuota{},
 		&ResourceQuotaList{},
 		&Namespace{},

pkg/api/types.go

@@ -1325,6 +1325,43 @@ type EnvVarSource struct {
 	SecretKeyRef *SecretKeySelector
 }
 
+// +nonNamespaced=true
+// +genclient=true
+
+// ResourceClass
+type ResourceClass struct {
+	metav1.TypeMeta
+	// +optional
+	metav1.ObjectMeta
+
+	// Spec defines resources required
+	// +optional
+	Spec ResourceClassSpec
+	// +optional
+	Status ResourceClassStatus
+}
+
+type ResourceClassStatus struct {
+	Allocatable int32
+	Request     int32
+}
+
+// Spec dictates features and properties of devices targeted by Resource Class
+type ResourceClassSpec struct {
+	// ResourceSelector  selects resources. ORed from each selector
+	ResourceSelector []ResourcePropertySelector
+	// +optional
+	SubResourcesCount int32
+}
+
+// RCList is a list of Rcs
+type ResourceClassList struct {
+	metav1.TypeMeta
+	// +optional
+	metav1.ListMeta
+	Items []ResourceClass
+}
+
 // ObjectFieldSelector selects an APIVersioned field of an object.
 type ObjectFieldSelector struct {
 	// Required: Version of the schema the FieldPath is written in terms of.
@@ -2859,6 +2896,44 @@ type NodeSystemInfo struct {
 	Architecture string
 }
 
+type DeviceSubResources struct {
+	// Name of the Device
+	Name string
+	// Count of devices
+	Quantity int32
+}
+
+// Device is a physical or logical device
+type Device struct {
+	metav1.TypeMeta
+	// +optional
+	metav1.ObjectMeta
+	// Count of devices
+	Quantity int32
+	// Device can be a group of several other devices
+	// +optional
+	SubResources DeviceSubResources
+}
+
+type ResourceSelectorOperator NodeSelectorOperator
+
+// A resource selector requirement is a selector that contains values, a key, and an operator
+// that relates the key and values
+type ResourceSelectorRequirement struct {
+	// The label key that the selector applies to
+	Key string
+	// Example 0.1, intel etc
+	Values []string
+	// operator
+	Operator ResourceSelectorOperator
+}
+
+// A null or empty selector matches no resources
+type ResourcePropertySelector struct {
+	// A list of resource/device selector requirements
+	MatchExpressions []ResourceSelectorRequirement
+}
+
 // NodeStatus is information about the current status of a node.
 type NodeStatus struct {
 	// Capacity represents the total resources of a node.
@@ -2891,6 +2966,12 @@ type NodeStatus struct {
 	// List of volumes that are attached to the node.
 	// +optional
 	VolumesAttached []AttachedVolume
+	// Total devices that are attached to the node.
+	// +optional
+	CapacityDevices []Device
+	// Allocatable devices that are attached to the node.
+	// +optional
+	AllocatableDevices []Device
 }
 
 type UniqueVolumeName string

pkg/api/v1/annotation_key_constants.go

@@ -34,6 +34,10 @@ const (
 	// in the Annotations of a Node.
 	TaintsAnnotationKey string = "scheduler.alpha.kubernetes.io/taints"
 
+	// ResClassPodAnnotationKeyPrefix represents the key of a device allocated
+	// to one container of a pod.
+	ResClassPodAnnotationKeyPrefix string = "scheduler.alpha.kubernetes.io/resClass"
+
 	// SeccompPodAnnotationKey represents the key of a seccomp profile applied
 	// to all containers of a pod.
 	SeccompPodAnnotationKey string = "seccomp.security.alpha.kubernetes.io/pod"

pkg/api/v1/helper/helpers.go

@@ -169,6 +169,40 @@ func containsAccessMode(modes []v1.PersistentVolumeAccessMode, mode v1.Persisten
 	return false
 }
 
+// NodeSelectorRequirementsAsSelector converts the []NodeSelectorRequirement api type into a struct that implements
+// labels.Selector.
+func ResourceSelectorRequirementsAsSelector(nsm []v1.ResourceSelectorRequirement) (labels.Selector, error) {
+	if len(nsm) == 0 {
+		return labels.Nothing(), nil
+	}
+	selector := labels.NewSelector()
+	for _, expr := range nsm {
+		var op selection.Operator
+		switch expr.Operator {
+		case v1.ResourceSelectorOpIn:
+			op = selection.In
+		case v1.ResourceSelectorOpNotIn:
+			op = selection.NotIn
+		case v1.ResourceSelectorOpExists:
+			op = selection.Exists
+		case v1.ResourceSelectorOpDoesNotExist:
+			op = selection.DoesNotExist
+		case v1.ResourceSelectorOpGt:
+			op = selection.GreaterThan
+		case v1.ResourceSelectorOpLt:
+			op = selection.LessThan
+		default:
+			return nil, fmt.Errorf("%q is not a valid node selector operator", expr.Operator)
+		}
+		r, err := labels.NewRequirement(expr.Key, op, expr.Values)
+		if err != nil {
+			return nil, err
+		}
+		selector = selector.Add(*r)
+	}
+	return selector, nil
+}
+
 // NodeSelectorRequirementsAsSelector converts the []NodeSelectorRequirement api type into a struct that implements
 // labels.Selector.
 func NodeSelectorRequirementsAsSelector(nsm []v1.NodeSelectorRequirement) (labels.Selector, error) {

pkg/api/v1/register.go

@@ -72,6 +72,9 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&List{},
 		&LimitRange{},
 		&LimitRangeList{},
+		&ResourceClass{},
+		&ResourceClassList{},
+		&Device{},
 		&ResourceQuota{},
 		&ResourceQuotaList{},
 		&Namespace{},

pkg/api/v1/types.go

@@ -1421,6 +1421,50 @@ type EnvVarSource struct {
 	SecretKeyRef *SecretKeySelector `json:"secretKeyRef,omitempty" protobuf:"bytes,4,opt,name=secretKeyRef"`
 }
 
+// +nonNamespaced=true
+// +genclient=true
+
+// ResourceClass is a resource class
+type ResourceClass struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard object's metadata.
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+
+	// Spec defines resources required
+	// +optional
+	Spec ResourceClassSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`
+	// Most recently observed status of the resource class.
+	// Populated by the system.
+	// Read-only
+	// +optional
+	Status ResourceClassStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"`
+}
+
+// Spec defines resources required
+type ResourceClassSpec struct {
+	// Resource Selector selects resources
+	ResourceSelector []ResourcePropertySelector `json:"resourceSelector" protobuf:"bytes,1,rep,name=resourceSelector"`
+	// +optional
+	SubResourcesCount int32 `json:"subResourcesCount" protobuf:"varint,2,opt,name=subResourcesCount"`
+}
+
+// ResourceClassStatus  is information about the current status of a resource class
+type ResourceClassStatus struct {
+	Allocatable int32 `json:"allocatable" protobuf:"varint,1,name=allocatable"`
+	Request     int32 `json:"request" protobuf:"varint,2,name=request"`
+}
+
+// ResourceClassList is list of rcs
+type ResourceClassList struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard list metadata.
+	// +optional
+	metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+	// List of nodes
+	Items []ResourceClass `json:"items" protobuf:"bytes,2,rep,name=items"`
+}
+
 // ObjectFieldSelector selects an APIVersioned field of an object.
 type ObjectFieldSelector struct {
 	// Version of the schema the FieldPath is written in terms of, defaults to "v1".
@@ -3279,6 +3323,60 @@ type NodeSystemInfo struct {
 	Architecture string `json:"architecture" protobuf:"bytes,10,opt,name=architecture"`
 }
 
+type DeviceSubResources struct {
+	// Name of the Device
+	Name string `json:"name" protobuf:"varint,1,name=name"`
+	// Count of devices
+	Quantity int32 `json:"quantity" protobuf:"varint,2,name=quantity"`
+}
+
+// +nonNamespaced=true
+
+// ResourceClass is a resource class
+// Device is a physical or logical device
+type Device struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard object's metadata.
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+	// count of such devices on node
+	Quantity int32 `json:"quantity" protobuf:"varint,2,name=quantity"`
+	// Device can be a group of several other devices
+	// +optional
+	SubResources DeviceSubResources `json:"SubResources" protobuf:"bytes,3,opt,name=SubResources,casttype=DeviceSubResources"`
+}
+
+type ResourceSelectorOperator string
+
+const (
+	ResourceSelectorOpIn           ResourceSelectorOperator = "In"
+	ResourceSelectorOpNotIn        ResourceSelectorOperator = "NotIn"
+	ResourceSelectorOpExists       ResourceSelectorOperator = "Exists"
+	ResourceSelectorOpDoesNotExist ResourceSelectorOperator = "DoesNotExist"
+	ResourceSelectorOpGt           ResourceSelectorOperator = "Gt"
+	ResourceSelectorOpLt           ResourceSelectorOperator = "Lt"
+)
+
+// A resource selector requirement is a selector that contains values, a key, and an operator
+// that relates the key and values
+type ResourceSelectorRequirement struct {
+	// The label key that the selector applies to
+	// +patchMergeKey=key
+	// +patchStrategy=merge
+	Key string `json:"key" patchStrategy:"merge" patchMergeKey:"key" protobuf:"bytes,1,opt,name=key"`
+	// Example 0.1, intel etc
+	// +optional
+	Values []string `json:"values,omitempty" protobuf:"bytes,2,rep,name=values"`
+	// operator
+	Operator ResourceSelectorOperator `json:"operator" protobuf:"bytes,3,opt,name=operator,casttype=ResourceSelectorOperator"`
+}
+
+// A null or empty selector matches no resources
+type ResourcePropertySelector struct {
+	// A list of resource/device selector requirements. ANDed from each ResourceSelectorRequirement
+	MatchExpressions []ResourceSelectorRequirement `json:"matchExpressions" protobuf:"bytes,1,rep,name=matchExpressions"`
+}
+
 // NodeStatus is information about the current status of a node.
 type NodeStatus struct {
 	// Capacity represents the total resources of a node.
@@ -3323,6 +3421,12 @@ type NodeStatus struct {
 	// List of volumes that are attached to the node.
 	// +optional
 	VolumesAttached []AttachedVolume `json:"volumesAttached,omitempty" protobuf:"bytes,10,rep,name=volumesAttached"`
+	// Total devices that are attached to the node.
+	// +optional
+	CapacityDevices []Device `json:"deviceCapacity,omitempty" protobuf:"bytes,11,rep,name=deviceCapacity"`
+	// Allocatable devices that are attached to the node.
+	// +optional
+	AllocatableDevices []Device `json:"deviceAllocatable,omitempty" protobuf:"bytes,12,rep,name=deviceAllocatable"`
 }
 
 type UniqueVolumeName string

pkg/api/v1/validation/validation.go

@@ -57,7 +57,8 @@ func ValidateResourceRequirements(requirements *v1.ResourceRequirements, fldPath
 	for resourceName, quantity := range requirements.Requests {
 		fldPath := reqPath.Key(string(resourceName))
 		// Validate resource name.
-		allErrs = append(allErrs, validateContainerResourceName(string(resourceName), fldPath)...)
+		//TODO(vikasc): commenting resource name validation because need to add handling for resource class name validation. One way, though very restrictive, could be to prefix rClass name with special string like "res-class-". For now switching off validation.
+		//allErrs = append(allErrs, validateContainerResourceName(string(resourceName), fldPath)...)
 		// Validate resource quantity.
 		allErrs = append(allErrs, ValidateResourceQuantityValue(string(resourceName), quantity, fldPath)...)
 	}