diff --git a/PROJECT b/PROJECT index b0aa53973d..622a843393 100644 --- a/PROJECT +++ b/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: x-k8s.io layout: - go.kubebuilder.io/v3 @@ -61,4 +65,12 @@ resources: kind: AdmissionCheck path: sigs.k8s.io/kueue/apis/kueue/v1beta1 version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + domain: x-k8s.io + group: kueue + kind: Cohort + path: sigs.k8s.io/kueue/apis/kueue/v1alpha1 + version: v1alpha1 version: "3" diff --git a/apis/kueue/v1alpha1/cohort_types.go b/apis/kueue/v1alpha1/cohort_types.go new file mode 100644 index 0000000000..1450bd9573 --- /dev/null +++ b/apis/kueue/v1alpha1/cohort_types.go @@ -0,0 +1,105 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kueuebeta "sigs.k8s.io/kueue/apis/kueue/v1beta1" +) + +// CohortSpec defines the desired state of Cohort +type CohortSpec struct { + // Parent references the name of the Cohort's parent, if + // any. It satisfies one of three cases: + // 1) Unset. This Cohort is the root of its Cohort tree. + // 2) References a non-existent Cohort. We use default Cohort (no borrowing/lending limits). + // 3) References an existent Cohort. + // + // If a cycle is created, we disable all members of the + // Cohort, including ClusterQueues, until the cycle is + // removed. We prevent further admission while the cycle + // exists. + // + //+kubebuilder:validation:MaxLength=253 + //+kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + // + Parent string `json:"parent,omitempty"` + + // ResourceGroups describes groupings of Resources and + // Flavors. Each ResourceGroup defines a list of Resources + // and a list of Flavors which provide quotas for these + // Resources. Each Resource and each Flavor may only form part + // of one ResourceGroup. There may be up to 16 ResourceGroups + // within a Cohort. + // + // BorrowingLimit limits how much members of this Cohort + // subtree can borrow from the parent subtree. + // + // LendingLimit limits how much members of this Cohort subtree + // can lend to the parent subtree. + // + // Borrowing and Lending limits must only be set when the + // Cohort has a parent. Otherwise, the Cohort create/update + // will be rejected by the webhook. + // + //+listType=atomic + //+kubebuilder:validation:MaxItems=16 + ResourceGroups []kueuebeta.ResourceGroup `json:"resourceGroups,omitempty"` +} + +const ( + // Condition indicating that a Cohort is correctly configured, + // for example, there is no cycle. + CohortActive = "Active" +) + +// CohortStatus defines the observed state of Cohort +type CohortStatus struct { + + //+listType=map + //+listMapKey=type + //+patchStrategy=merge + //+patchMergeKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:scope=Cluster + +// Cohort is the Schema for the cohorts API +type Cohort struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CohortSpec `json:"spec,omitempty"` + Status CohortStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CohortList contains a list of Cohort +type CohortList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Cohort `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Cohort{}, &CohortList{}) +} diff --git a/apis/kueue/v1alpha1/zz_generated.deepcopy.go b/apis/kueue/v1alpha1/zz_generated.deepcopy.go index 61cdc56186..7bce77a1e2 100644 --- a/apis/kueue/v1alpha1/zz_generated.deepcopy.go +++ b/apis/kueue/v1alpha1/zz_generated.deepcopy.go @@ -23,8 +23,112 @@ package v1alpha1 import ( "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/kueue/apis/kueue/v1beta1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Cohort) DeepCopyInto(out *Cohort) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cohort. +func (in *Cohort) DeepCopy() *Cohort { + if in == nil { + return nil + } + out := new(Cohort) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Cohort) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CohortList) DeepCopyInto(out *CohortList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Cohort, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CohortList. +func (in *CohortList) DeepCopy() *CohortList { + if in == nil { + return nil + } + out := new(CohortList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CohortList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CohortSpec) DeepCopyInto(out *CohortSpec) { + *out = *in + if in.ResourceGroups != nil { + in, out := &in.ResourceGroups, &out.ResourceGroups + *out = make([]v1beta1.ResourceGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CohortSpec. +func (in *CohortSpec) DeepCopy() *CohortSpec { + if in == nil { + return nil + } + out := new(CohortSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CohortStatus) DeepCopyInto(out *CohortStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CohortStatus. +func (in *CohortStatus) DeepCopy() *CohortStatus { + if in == nil { + return nil + } + out := new(CohortStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeConfig) DeepCopyInto(out *KubeConfig) { *out = *in diff --git a/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml b/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml new file mode 100644 index 0000000000..7be19cb006 --- /dev/null +++ b/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml @@ -0,0 +1,287 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: cohorts.kueue.x-k8s.io +spec: + group: kueue.x-k8s.io + names: + kind: Cohort + listKind: CohortList + plural: cohorts + singular: cohort + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Cohort is the Schema for the cohorts API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CohortSpec defines the desired state of Cohort + properties: + parent: + description: |- + Parent references the name of the Cohort's parent, if + any. It satisfies one of three cases: + 1) Unset. This Cohort is the root of its Cohort tree. + 2) References a non-existent Cohort. We use default Cohort (no borrowing/lending limits). + 3) References an existent Cohort. + + + If a cycle is created, we disable all members of the + Cohort, including ClusterQueues, until the cycle is + removed. We prevent further admission while the cycle + exists. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resourceGroups: + description: |- + ResourceGroups describes groupings of Resources and + Flavors. Each ResourceGroup defines a list of Resources + and a list of Flavors which provide quotas for these + Resources. Each Resource and each Flavor may only form part + of one ResourceGroup. There may be up to 16 ResourceGroups + within a Cohort. + + + BorrowingLimit limits how much members of this Cohort + subtree can borrow from the parent subtree. + + + LendingLimit limits how much members of this Cohort subtree + can lend to the parent subtree. + + + Borrowing and Lending limits must only be set when the + Cohort has a parent. Otherwise, the Cohort create/update + will be rejected by the webhook. + items: + properties: + coveredResources: + description: |- + coveredResources is the list of resources covered by the flavors in this + group. + Examples: cpu, memory, vendor.com/gpu. + The list cannot be empty and it can contain up to 16 resources. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + minItems: 1 + type: array + flavors: + description: |- + flavors is the list of flavors that provide the resources of this group. + Typically, different flavors represent different hardware models + (e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot + cpus). + Each flavor MUST list all the resources listed for this group in the same + order as the .resources field. + The list cannot be empty and it can contain up to 16 flavors. + items: + properties: + name: + description: |- + name of this flavor. The name should match the .metadata.name of a + ResourceFlavor. If a matching ResourceFlavor does not exist, the + ClusterQueue will have an Active condition set to False. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: |- + resources is the list of quotas for this flavor per resource. + There could be up to 16 resources. + items: + properties: + borrowingLimit: + anyOf: + - type: integer + - type: string + description: |- + borrowingLimit is the maximum amount of quota for the [flavor, resource] + combination that this ClusterQueue is allowed to borrow from the unused + quota of other ClusterQueues in the same cohort. + In total, at a given time, Workloads in a ClusterQueue can consume a + quantity of quota equal to nominalQuota+borrowingLimit, assuming the other + ClusterQueues in the cohort have enough unused quota. + If null, it means that there is no borrowing limit. + If not null, it must be non-negative. + borrowingLimit must be null if spec.cohort is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + lendingLimit: + anyOf: + - type: integer + - type: string + description: |- + lendingLimit is the maximum amount of unused quota for the [flavor, resource] + combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. + In total, at a given time, ClusterQueue reserves for its exclusive use + a quantity of quota equals to nominalQuota - lendingLimit. + If null, it means that there is no lending limit, meaning that + all the nominalQuota can be borrowed by other clusterQueues in the cohort. + If not null, it must be non-negative. + lendingLimit must be null if spec.cohort is empty. + This field is in alpha stage. To be able to use this field, + enable the feature gate LendingLimit, which is disabled by default. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name of this resource. + type: string + nominalQuota: + anyOf: + - type: integer + - type: string + description: |- + nominalQuota is the quantity of this resource that is available for + Workloads admitted by this ClusterQueue at a point in time. + The nominalQuota must be non-negative. + nominalQuota should represent the resources in the cluster available for + running jobs (after discounting resources consumed by system components + and pods not managed by kueue). In an autoscaled cluster, nominalQuota + should account for resources that can be provided by a component such as + Kubernetes cluster-autoscaler. + + + If the ClusterQueue belongs to a cohort, the sum of the quotas for each + (flavor, resource) combination defines the maximum quantity that can be + allocated by a ClusterQueue in the cohort. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + - nominalQuota + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - coveredResources + - flavors + type: object + x-kubernetes-validations: + - message: flavors must have the same number of resources as the + coveredResources + rule: self.flavors.all(x, size(x.resources) == size(self.coveredResources)) + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + status: + description: CohortStatus defines the observed state of Cohort + properties: + conditions: + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/site/content/en/docs/reference/kueue-alpha.v1alpha1.md b/site/content/en/docs/reference/kueue-alpha.v1alpha1.md index 7620453cd3..e759b679f2 100644 --- a/site/content/en/docs/reference/kueue-alpha.v1alpha1.md +++ b/site/content/en/docs/reference/kueue-alpha.v1alpha1.md @@ -74,6 +74,116 @@ description: Generated API reference documentation for kueue.x-k8s.io/v1alpha1. +## `Cohort` {#kueue-x-k8s-io-v1alpha1-Cohort} + + +**Appears in:** + + + +

Cohort is the Schema for the cohorts API

+ + + + + + + + + + + + + + +
FieldDescription
spec [Required]
+CohortSpec +		 + No description provided.
status [Required]
+CohortStatus +		 + No description provided.
+ +## `CohortSpec` {#kueue-x-k8s-io-v1alpha1-CohortSpec} + + +**Appears in:** + +- [Cohort](#kueue-x-k8s-io-v1alpha1-Cohort) + + +

CohortSpec defines the desired state of Cohort

+ + + + + + + + + + + + + + +
FieldDescription
parent [Required]
+string +		 +

Parent references the name of the Cohort's parent, if +any. It satisfies one of three cases:

+
    +
  1. Unset. This Cohort is the root of its Cohort tree.
    2. +
  2. References a non-existent Cohort. We use default Cohort (no borrowing/lending limits).
    3. +
  3. References an existent Cohort.
    4. +
+

If a cycle is created, we disable all members of the +Cohort, including ClusterQueues, until the cycle is +removed. We prevent further admission while the cycle +exists.

+
resourceGroups [Required]
+[]ResourceGroup +		 +

ResourceGroups describes groupings of Resources and +Flavors. Each ResourceGroup defines a list of Resources +and a list of Flavors which provide quotas for these +Resources. Each Resource and each Flavor may only form part +of one ResourceGroup. There may be up to 16 ResourceGroups +within a Cohort.

+

BorrowingLimit limits how much members of this Cohort +subtree can borrow from the parent subtree.

+

LendingLimit limits how much members of this Cohort subtree +can lend to the parent subtree.

+

Borrowing and Lending limits must only be set when the +Cohort has a parent. Otherwise, the Cohort create/update +will be rejected by the webhook.

+
+ +## `CohortStatus` {#kueue-x-k8s-io-v1alpha1-CohortStatus} + + +**Appears in:** + +- [Cohort](#kueue-x-k8s-io-v1alpha1-Cohort) + + +

CohortStatus defines the observed state of Cohort

+ + + + + + + + + + + +
FieldDescription
conditions [Required]
+[]k8s.io/apimachinery/pkg/apis/meta/v1.Condition +		 + No description provided.
+ ## `KubeConfig` {#kueue-x-k8s-io-v1alpha1-KubeConfig}