add simple Cluster scheduler implementation (#13)

* implement cluster scheduler

* fix scheduler setup and distinguish between cluster and clusterrequest label selectors

* add scheduler unit tests

* rename fields

* add 'local' selectors to cluster definitions in scheduler config

* add scheduler docs

* move cluster resources to platform cluster

* fix kubebuilder CRD generation annotations

* add scheduler cluster deletion and minor fixes

* add helper function for request phase

* enhance help text for --controllers flag

* fix task generate

* reduce cyclomatic complexity

Author: Diaphteiros

Taskfile.yaml

@@ -9,7 +9,7 @@ includes:
       NESTED_MODULES: api
       API_DIRS: '{{.ROOT_DIR}}/api/provider/v1alpha1/... {{.ROOT_DIR}}/api/clusters/v1alpha1/...'
       MANIFEST_OUT: '{{.ROOT_DIR}}/api/crds/manifests'
-      CODE_DIRS: '{{.ROOT_DIR}}/cmd/... {{.ROOT_DIR}}/internal/... {{.ROOT_DIR}}/test/... {{.ROOT_DIR}}/api/provider/v1alpha1/... {{.ROOT_DIR}}/api/clusters/v1alpha1/...'
+      CODE_DIRS: '{{.ROOT_DIR}}/cmd/... {{.ROOT_DIR}}/internal/... {{.ROOT_DIR}}/api/provider/v1alpha1/... {{.ROOT_DIR}}/api/clusters/v1alpha1/...'
       COMPONENTS: 'openmcp-operator'
       REPO_URL: 'https://github.com/openmcp-project/openmcp-operator'
       GENERATE_DOCS_INDEX: "true"

api/clusters/v1alpha1/accessrequest_types.go

@@ -12,6 +12,12 @@ type AccessRequestSpec struct {
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="clusterRef is immutable"
 	ClusterRef NamespacedObjectReference `json:"clusterRef"`
 
+	// RequestRef is the reference to the ClusterRequest for whose Cluster access is requested.
+	// Exactly one of clusterRef or requestRef must be set.
+	// This value is immutable.
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="requestRef is immutable"
+	RequestRef NamespacedObjectReference `json:"requestRef"`
+
 	// Permissions are the requested permissions.
 	Permissions []PermissionsRequest `json:"permissions"`
 }
@@ -32,14 +38,20 @@ type AccessRequestStatus struct {
 	CommonStatus `json:",inline"`
 
 	// Phase is the current phase of the request.
+	// +kubebuilder:default=Pending
+	// +kubebuilder:validation:Enum=Pending;Granted;Denied
 	Phase RequestPhase `json:"phase"`
 
-	// TODO: expose actual access information
+	// SecretRef holds the reference to the secret that contains the actual credentials.
+	SecretRef *NamespacedObjectReference `json:"secretRef,omitempty"`
 }
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
-// +kubebuilder:metadata:labels="openmcp.cloud/cluster=onboarding"
+// +kubebuilder:resource:shortName=ar;areq
+// +kubebuilder:metadata:labels="openmcp.cloud/cluster=platform"
+// +kubebuilder:selectablefield:JSONPath=".status.phase"
+// +kubebuilder:printcolumn:JSONPath=".status.phase",name="Phase",type=string
 
 // AccessRequest is the Schema for the accessrequests API
 type AccessRequest struct {

api/clusters/v1alpha1/cluster_types.go

@@ -6,6 +6,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 // ClusterSpec defines the desired state of Cluster
@@ -81,15 +82,14 @@ const (
 
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
-// +kubebuilder:metadata:labels="openmcp.cloud/cluster=onboarding"
-// +kubebuilder:selectablefield:JSONPath=".spec.clusterProfileRef.name"
+// +kubebuilder:metadata:labels="openmcp.cloud/cluster=platform"
+// +kubebuilder:selectablefield:JSONPath=".spec.profile"
 // +kubebuilder:printcolumn:JSONPath=".spec.purposes",name="Purposes",type=string
 // +kubebuilder:printcolumn:JSONPath=`.status.phase`,name="Phase",type=string
 // +kubebuilder:printcolumn:JSONPath=`.metadata.annotations["clusters.openmcp.cloud/k8sversion"]`,name="Version",type=string
 // +kubebuilder:printcolumn:JSONPath=`.metadata.annotations["clusters.openmcp.cloud/profile"]`,name="Profile",type=string
-// +kubebuilder:printcolumn:JSONPath=`.metadata.labels["environment.clusters.openmcp.cloud"]`,name="Env",type=string,priority=10
 // +kubebuilder:printcolumn:JSONPath=`.metadata.labels["provider.clusters.openmcp.cloud"]`,name="Provider",type=string, priority=10
-// +kubebuilder:printcolumn:JSONPath=".spec.clusterProfileRef.name",name="ProfileRef",type=string,priority=10
+// +kubebuilder:printcolumn:JSONPath=".spec.profile",name="ProfileRef",type=string,priority=10
 // +kubebuilder:printcolumn:JSONPath=`.metadata.annotations["clusters.openmcp.cloud/providerinfo"]`,name="Info",type=string,priority=10
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 
@@ -132,12 +132,19 @@ func (cs *ClusterStatus) SetProviderStatus(from any) error {
 
 // GetTenancyCount returns the number of ClusterRequests currently pointing to this cluster.
 // This is determined by counting the finalizers that have the corresponding prefix.
+// Note that only unique finalizers are counted, so if there are multiple identical request finalizers
+// (which should not happen), this method's return value might not match the actual number of finalizers with the prefix.
 func (c *Cluster) GetTenancyCount() int {
-	count := 0
+	return c.GetRequestUIDs().Len()
+}
+
+// GetRequestUIDs returns the UIDs of all ClusterRequests that have marked this cluster with a corresponding finalizer.
+func (c *Cluster) GetRequestUIDs() sets.Set[string] {
+	res := sets.New[string]()
 	for _, fin := range c.Finalizers {
 		if strings.HasPrefix(fin, RequestFinalizerOnClusterPrefix) {
-			count++
+			res.Insert(strings.TrimPrefix(fin, RequestFinalizerOnClusterPrefix))
 		}
 	}
-	return count
+	return res
 }

api/clusters/v1alpha1/clusterprofile_types.go

@@ -6,9 +6,6 @@ import (
 
 // ClusterProfileSpec defines the desired state of Provider.
 type ClusterProfileSpec struct {
-	// Environment is the environment in which the ClusterProvider resides.
-	Environment string `json:"environment"`
-
 	// ProviderRef is a reference to the ClusterProvider
 	ProviderRef ObjectReference `json:"providerRef"`
 
@@ -30,11 +27,9 @@ type SupportedK8sVersion struct {
 
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:scope=Cluster,shortName=cprof;profile
-// +kubebuilder:metadata:labels="openmcp.cloud/cluster=onboarding"
-// +kubebuilder:selectablefield:JSONPath=".spec.environment"
+// +kubebuilder:metadata:labels="openmcp.cloud/cluster=platform"
 // +kubebuilder:selectablefield:JSONPath=".spec.providerRef.name"
 // +kubebuilder:selectablefield:JSONPath=".spec.providerConfigRef.name"
-// +kubebuilder:printcolumn:JSONPath=".spec.environment",name="Env",type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.providerRef.name",name="Provider",type=string
 // +kubebuilder:printcolumn:JSONPath=".spec.providerConfigRef.name",name="Config",type=string
 

api/clusters/v1alpha1/clusterrequest_types.go

@@ -16,20 +16,35 @@ type ClusterRequestStatus struct {
 	CommonStatus `json:",inline"`
 
 	// Phase is the current phase of the request.
+	// +kubebuilder:default=Pending
+	// +kubebuilder:validation:Enum=Pending;Granted;Denied
 	Phase RequestPhase `json:"phase"`
 
-	// ClusterRef is the reference to the Cluster that was returned as a result of a granted request.
+	// Cluster is the reference to the Cluster that was returned as a result of a granted request.
 	// Note that this information needs to be recoverable in case this status is lost, e.g. by adding a back reference in form of a finalizer to the Cluster resource.
 	// +optional
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="clusterRef is immutable"
-	ClusterRef *NamespacedObjectReference `json:"clusterRef,omitempty"`
+	Cluster *NamespacedObjectReference `json:"clusterRef,omitempty"`
 }
 
 type RequestPhase string
 
+func (p RequestPhase) IsGranted() bool {
+	return p == REQUEST_GRANTED
+}
+
+func (p RequestPhase) IsDenied() bool {
+	return p == REQUEST_DENIED
+}
+
+func (p RequestPhase) IsPending() bool {
+	return p == "" || p == REQUEST_PENDING
+}
+
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
-// +kubebuilder:metadata:labels="openmcp.cloud/cluster=onboarding"
+// +kubebuilder:resource:shortName=cr;creq
+// +kubebuilder:metadata:labels="openmcp.cloud/cluster=platform"
 // +kubebuilder:selectablefield:JSONPath=".spec.purpose"
 // +kubebuilder:selectablefield:JSONPath=".status.phase"
 // +kubebuilder:printcolumn:JSONPath=".spec.purpose",name="Purpose",type=string

api/clusters/v1alpha1/constants.go

@@ -75,6 +75,10 @@ const (
 	EnvironmentAnnotation = "clusters.openmcp.cloud/environment"
 	// ProviderAnnotation can be used to display the provider of the cluster.
 	ProviderAnnotation = "clusters.openmcp.cloud/provider"
+
+	// DeleteWithoutRequestsLabel marks that the corresponding cluster can be deleted if the scheduler removes the last request pointing to it.
+	// Its value must be "true" for the label to take effect.
+	DeleteWithoutRequestsLabel = "clusters.openmcp.cloud/delete-without-requests"
 )
 
 const (

api/clusters/v1alpha1/zz_generated.deepcopy.go

@@ -5,18 +5,25 @@ metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.17.3
   labels:
-    openmcp.cloud/cluster: onboarding
+    openmcp.cloud/cluster: platform
   name: accessrequests.clusters.openmcp.cloud
 spec:
   group: clusters.openmcp.cloud
   names:
     kind: AccessRequest
     listKind: AccessRequestList
     plural: accessrequests
+    shortNames:
+    - ar
+    - areq
     singular: accessrequest
   scope: Namespaced
   versions:
-  - name: v1alpha1
+  - additionalPrinterColumns:
+    - jsonPath: .status.phase
+      name: Phase
+      type: string
+    name: v1alpha1
     schema:
       openAPIV3Schema:
         description: AccessRequest is the Schema for the accessrequests API
@@ -125,9 +132,30 @@ spec:
                   - rules
                   type: object
                 type: array
+              requestRef:
+                description: |-
+                  RequestRef is the reference to the ClusterRequest for whose Cluster access is requested.
+                  Exactly one of clusterRef or requestRef must be set.
+                  This value is immutable.
+                properties:
+                  name:
+                    description: Name is the name of the referenced resource.
+                    minLength: 1
+                    type: string
+                  namespace:
+                    description: Namespace is the namespace of the referenced resource.
+                    type: string
+                required:
+                - name
+                - namespace
+                type: object
+                x-kubernetes-validations:
+                - message: requestRef is immutable
+                  rule: self == oldSelf
             required:
             - clusterRef
             - permissions
+            - requestRef
             type: object
           status:
             description: AccessRequestStatus defines the observed state of AccessRequest
@@ -181,18 +209,40 @@ spec:
                 format: int64
                 type: integer
               phase:
+                default: Pending
                 description: Phase is the current phase of the request.
+                enum:
+                - Pending
+                - Granted
+                - Denied
                 type: string
               reason:
                 description: Reason is expected to contain a CamelCased string that
                   provides further information in a machine-readable format.
                 type: string
+              secretRef:
+                description: SecretRef holds the reference to the secret that contains
+                  the actual credentials.
+                properties:
+                  name:
+                    description: Name is the name of the referenced resource.
+                    minLength: 1
+                    type: string
+                  namespace:
+                    description: Namespace is the namespace of the referenced resource.
+                    type: string
+                required:
+                - name
+                - namespace
+                type: object
             required:
             - lastReconcileTime
             - observedGeneration
             - phase
             type: object
         type: object
+    selectableFields:
+    - jsonPath: .status.phase
     served: true
     storage: true
     subresources:

api/crds/manifests/clusters.openmcp.cloud_accessrequests.yaml

@@ -5,7 +5,7 @@ metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.17.3
   labels:
-    openmcp.cloud/cluster: onboarding
+    openmcp.cloud/cluster: platform
   name: clusterprofiles.clusters.openmcp.cloud
 spec:
   group: clusters.openmcp.cloud
@@ -20,9 +20,6 @@ spec:
   scope: Cluster
   versions:
   - additionalPrinterColumns:
-    - jsonPath: .spec.environment
-      name: Env
-      type: string
     - jsonPath: .spec.providerRef.name
       name: Provider
       type: string
@@ -53,10 +50,6 @@ spec:
           spec:
             description: ClusterProfileSpec defines the desired state of Provider.
             properties:
-              environment:
-                description: Environment is the environment in which the ClusterProvider
-                  resides.
-                type: string
               providerConfigRef:
                 description: ProviderConfigRef is a reference to the provider-specific
                   configuration.
@@ -94,14 +87,12 @@ spec:
                   type: object
                 type: array
             required:
-            - environment
             - providerConfigRef
             - providerRef
             - supportedVersions
             type: object
         type: object
     selectableFields:
-    - jsonPath: .spec.environment
     - jsonPath: .spec.providerRef.name
     - jsonPath: .spec.providerConfigRef.name
     served: true

api/crds/manifests/clusters.openmcp.cloud_clusterprofiles.yaml

@@ -5,14 +5,17 @@ metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.17.3
   labels:
-    openmcp.cloud/cluster: onboarding
+    openmcp.cloud/cluster: platform
   name: clusterrequests.clusters.openmcp.cloud
 spec:
   group: clusters.openmcp.cloud
   names:
     kind: ClusterRequest
     listKind: ClusterRequestList
     plural: clusterrequests
+    shortNames:
+    - cr
+    - creq
     singular: clusterrequest
   scope: Namespaced
   versions:
@@ -61,7 +64,7 @@ spec:
             properties:
               clusterRef:
                 description: |-
-                  ClusterRef is the reference to the Cluster that was returned as a result of a granted request.
+                  Cluster is the reference to the Cluster that was returned as a result of a granted request.
                   Note that this information needs to be recoverable in case this status is lost, e.g. by adding a back reference in form of a finalizer to the Cluster resource.
                 properties:
                   name:
@@ -127,7 +130,12 @@ spec:
                 format: int64
                 type: integer
               phase:
+                default: Pending
                 description: Phase is the current phase of the request.
+                enum:
+                - Pending
+                - Granted
+                - Denied
                 type: string
               reason:
                 description: Reason is expected to contain a CamelCased string that