Add opt-in P&F configs for Visibility API and reference from documentation

Makefile

@@ -265,6 +265,8 @@ artifacts: kustomize yq helm ## Generate release artifacts.
 	$(KUSTOMIZE) build config/dev -o artifacts/manifests-dev.yaml
 	$(KUSTOMIZE) build config/alpha-enabled -o artifacts/manifests-alpha-enabled.yaml
 	$(KUSTOMIZE) build config/prometheus -o artifacts/prometheus.yaml
+	$(KUSTOMIZE) build config/visibility-apf/default -o artifacts/visibility-apf.yaml
+	$(KUSTOMIZE) build config/visibility-apf/1_28 -o artifacts/visibility-apf-1-28.yaml
 	@$(call clean-manifests)
 	# Update the image tag and policy
 	$(YQ)  e  '.controllerManager.manager.image.repository = "$(IMAGE_REPO)" | .controllerManager.manager.image.tag = "$(GIT_TAG)" | .controllerManager.manager.image.pullPolicy = "IfNotPresent"' -i charts/kueue/values.yaml

charts/kueue/README.md

@@ -48,6 +48,7 @@ The following table lists the configurable parameters of the kueue chart and the
 | `fullnameOverride`                                     | override the resource name                             | ``                                          |
 | `enablePrometheus`                                     | enable Prometheus                                      | `false`                                     |
 | `enableCertManager`                                    | enable CertManager                                     | `false`                                     |
+| `enableVisibilityAPF`                                  | enable APF for the visibility API                      | `false`                                     |
 | `controllerManager.kubeRbacProxy.image`                | controllerManager.kubeRbacProxy's image                | `gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0` |
 | `controllerManager.manager.image`                      | controllerManager.manager's image                      | `us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue` |
 | `controllerManager.manager.tag`                        | controllerManager.manager's tag                        | `main`                                      |

charts/kueue/templates/visibility-apf/flowschema.yaml

@@ -0,0 +1,33 @@
+{{- if .Values.enableVisibilityAPF }}
+apiVersion: flowcontrol.apiserver.k8s.io/{{ and (eq .Capabilities.KubeVersion.Major "1") (eq .Capabilities.KubeVersion.Minor "28") | ternary "v1beta3" "v1" }}
+kind: FlowSchema
+metadata:
+  labels:
+  {{- include "kueue.labels" . | nindent 4 }}
+  name: '{{ include "kueue.fullname" . }}-visibility'
+  namespace: '{{ .Release.Namespace }}'
+spec:
+  distinguisherMethod:
+    type: ByUser
+  matchingPrecedence: 9000
+  priorityLevelConfiguration:
+    name: kueue-visibility
+  rules:
+    - resourceRules:
+        - apiGroups:
+            - 'visibility.kueue.x-k8s.io'
+          clusterScope: true
+          namespaces:
+            - '*'
+          resources:
+            - '*'
+          verbs:
+            - '*'
+      subjects:
+        - group:
+            name: system:unauthenticated
+          kind: Group
+        - group:
+            name: system:authenticated
+          kind: Group
+{{- end }}

charts/kueue/templates/visibility-apf/prioritylevelconfigurations.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.enableVisibilityAPF }}
+apiVersion: flowcontrol.apiserver.k8s.io/{{ and (eq .Capabilities.KubeVersion.Major "1") (eq .Capabilities.KubeVersion.Minor "28") | ternary "v1beta3" "v1" }}
+kind: PriorityLevelConfiguration
+metadata:
+  labels:
+  {{- include "kueue.labels" . | nindent 4 }}
+  name: '{{ include "kueue.fullname" . }}-visibility'
+  namespace: '{{ .Release.Namespace }}'
+spec:
+  limited:
+    lendablePercent: 90
+    limitResponse:
+      queuing:
+        handSize: 4
+        queueLengthLimit: 50
+        queues: 16
+      type: Queue
+    nominalConcurrencyShares: 10
+  type: Limited
+{{- end }}

charts/kueue/values.yaml

@@ -7,6 +7,8 @@ fullnameOverride: ""
 enablePrometheus: false
 # Enable x509 automated certificate management using cert-manager (cert-manager.io)
 enableCertManager: false
+# Enable API Priority and Fairness configuration for the visibility API
+enableVisibilityAPF: false
 # Customize controllerManager
 controllerManager:
   #featureGates:

config/components/visibility-apf/flowschema.yaml

@@ -0,0 +1,29 @@
+apiVersion: flowcontrol.apiserver.k8s.io/v1
+kind: FlowSchema
+metadata:
+  name: visibility
+  namespace: system
+spec:
+  distinguisherMethod:
+    type: ByUser
+  matchingPrecedence: 9000
+  priorityLevelConfiguration:
+    name: kueue-visibility
+  rules:
+    - resourceRules:
+        - apiGroups:
+            - 'visibility.kueue.x-k8s.io'
+          clusterScope: true
+          namespaces:
+            - '*'
+          resources:
+            - '*'
+          verbs:
+            - '*'
+      subjects:
+        - group:
+            name: system:unauthenticated
+          kind: Group
+        - group:
+            name: system:authenticated
+          kind: Group

config/components/visibility-apf/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- flowschema.yaml
+- prioritylevelconfigurations.yaml

config/components/visibility-apf/prioritylevelconfigurations.yaml

@@ -0,0 +1,16 @@
+apiVersion: flowcontrol.apiserver.k8s.io/v1
+kind: PriorityLevelConfiguration
+metadata:
+  name: visibility
+  namespace: system
+spec:
+  limited:
+    lendablePercent: 90
+    limitResponse:
+      queuing:
+        handSize: 4
+        queueLengthLimit: 50
+        queues: 16
+      type: Queue
+    nominalConcurrencyShares: 10
+  type: Limited

config/visibility-apf/1_28/kustomization.yaml

@@ -0,0 +1,19 @@
+# This overlay builds the APF components for Kubernetes version 1.28 to be used in combination
+# with other overlays.
+
+resources:
+- ../default/
+
+patches:
+  - target:
+      kind: FlowSchema
+    patch: |
+      - op: replace
+        path: /apiVersion
+        value: flowcontrol.apiserver.k8s.io/v1beta3
+  - target:
+      kind: PriorityLevelConfiguration
+    patch: |
+      - op: replace
+        path: /apiVersion
+        value: flowcontrol.apiserver.k8s.io/v1beta3

config/visibility-apf/default/kustomization.yaml

@@ -0,0 +1,14 @@
+# This overlay builds the APF components to be used in combination
+# with other overlays.
+
+namespace: kueue-system
+namePrefix: kueue-
+
+labels:
+  - pairs:
+      app.kubernetes.io/name: kueue
+      app.kubernetes.io/component: controller
+      control-plane: controller-manager
+
+resources:
+  - ../../components/visibility-apf/

hack/e2e-common.sh

@@ -19,6 +19,8 @@ export GINKGO="$ROOT_DIR"/bin/ginkgo
 export KIND="$ROOT_DIR"/bin/kind
 export YQ="$ROOT_DIR"/bin/yq
 
+export KIND_VERSION="${E2E_KIND_VERSION/"kindest/node:v"/}"
+
 export JOBSET_MANIFEST="https://github.com/kubernetes-sigs/jobset/releases/download/${JOBSET_VERSION}/manifests.yaml"
 export JOBSET_IMAGE=registry.k8s.io/jobset/jobset:${JOBSET_VERSION}
 export JOBSET_CRDS=${ROOT_DIR}/dep-crds/jobset-operator/
@@ -75,7 +77,11 @@ function cluster_kind_load_image {
 # $1 cluster
 function cluster_kueue_deploy {
     kubectl config use-context "kind-${1}"
-    kubectl apply --server-side -k test/e2e/config
+    if [ "${KIND_VERSION%.*}" = "1.28" ]; then
+        kubectl apply --server-side -k test/e2e/config/1_28
+    else
+        kubectl apply --server-side -k test/e2e/config/default
+    fi
 }
 
 #$1 - cluster name

hack/multikueue-e2e-test.sh

@@ -51,8 +51,7 @@ function startup {
         if [ ! -d "$ARTIFACTS" ]; then
             mkdir -p "$ARTIFACTS"
         fi
-
-        KIND_VERSION=${E2E_KIND_VERSION/"kindest/node:v"/}
+
         MANAGER_KIND_CONFIG="${SOURCE_DIR}/multikueue/manager-cluster.kind-${KIND_VERSION}.yaml"
         if [ !  -f "$MANAGER_KIND_CONFIG" ]; then
             MANAGER_KIND_CONFIG="${SOURCE_DIR}/multikueue/manager-cluster.kind.yaml"

hack/update-helm.sh

@@ -23,11 +23,13 @@ SRC_CRD_DIR=config/components/crd/bases
 SRC_RBAC_DIR=config/components/rbac
 SRC_WEBHOOK_DIR=config/components/webhook
 SRC_VISIBILITY_DIR=config/components/visibility
+SRC_VISIBILITY_APF_DIR=config/components/visibility-apf
 
 DEST_CRD_DIR=charts/kueue/templates/crd
 DEST_RBAC_DIR=charts/kueue/templates/rbac
 DEST_WEBHOOK_DIR=charts/kueue/templates/webhook
 DEST_VISIBILITY_DIR=charts/kueue/templates/visibility
+DEST_VISIBILITY_APF_DIR=charts/kueue/templates/visibility-apf
 
 YQ=./bin/yq
 SED=${SED:-/usr/bin/sed}
@@ -48,6 +50,8 @@ find $SRC_RBAC_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_RBAC_D
 find $SRC_WEBHOOK_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_WEBHOOK_DIR \;
 # shellcheck disable=SC2086
 find $SRC_VISIBILITY_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_VISIBILITY_DIR \;
+# shellcheck disable=SC2086
+find $SRC_VISIBILITY_APF_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_VISIBILITY_APF_DIR \;
 $YQ -N -s '.kind' ${DEST_WEBHOOK_DIR}/manifests.yaml
 rm ${DEST_WEBHOOK_DIR}/manifests.yaml
 files=("MutatingWebhookConfiguration.yml" "ValidatingWebhookConfiguration.yml")
@@ -298,3 +302,17 @@ EOT
   fi
   $SED -i '/^metadata:.*/a\  labels:\n  {{- include "kueue.labels" . | nindent 4 }}' "$output_file"
 done
+
+# Replace flowcontrol version on visibility-apf directory
+for output_file in "${DEST_VISIBILITY_APF_DIR}"/*.yaml; do
+  $YQ -N -i '.apiVersion = "flowcontrol.apiserver.k8s.io/{{ and (eq .Capabilities.KubeVersion.Major \"1\") (eq .Capabilities.KubeVersion.Minor \"28\") | ternary \"v1beta3\" \"v1\" }}"' "$output_file"
+  $YQ -N -i '.metadata.name |= "{{ include \"kueue.fullname\" . }}-" + .' "$output_file"
+  $YQ -N -i '.metadata.namespace = "{{ .Release.Namespace }}"' "$output_file"
+  $SED -i '/^metadata:.*/a\  labels:\n  {{- include "kueue.labels" . | nindent 4 }}' "$output_file"
+  {
+    echo '{{- if .Values.enableVisibilityAPF }}'
+    cat "$output_file"
+    echo "{{- end }}"
+  } > "${output_file}.tmp"
+  mv "${output_file}.tmp" "${output_file}"
+done

site/content/en/docs/installation/_index.md

@@ -16,6 +16,7 @@ description: >
   - [Uninstall](#uninstall-1)
 - [Build and install from source](#build-and-install-from-source)
   - [Add metrics scraping for prometheus-operator](#add-metrics-scraping-for-prometheus-operator-1)
+  - [Add API Priority and Fairness configuration for the visibility API](#add-api-priority-and-fairness-configuration-for-the-visibility-api)
   - [Uninstall](#uninstall-2)
 - [Install via Helm](#install-via-helm)
 - [Change the feature gates configuration](#change-the-feature-gates-configuration)
@@ -74,6 +75,10 @@ We can follow `https://prometheus-operator.dev/docs/prologue/quick-start/` to in
 kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/{{< param "version" >}}/prometheus.yaml
 ```
 
+### Add API Priority and Fairness configuration for the visibility API
+
+See [Configure API Priority and Fairness](/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/#configure-api-priority-and-fairness) for more details.
+
 ### Uninstall
 
 To uninstall a released version of Kueue from your cluster, run the following command:

site/content/en/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand.md

@@ -26,6 +26,15 @@ Make sure the following conditions are met:
 
 VisibilityOnDemand is an `Alpha` feature disabled by default. To use the visibility API  change [the feature gates configuration](/docs/installation/#change-the-feature-gates-configuration) and set `VisibilityOnDemand=true`.
 
+### Configure API Priority and Fairness:
+
+To install the [API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/) configuration for the visibility API apply one of the manifests, depending on your Kubernetes version:
+
+{{< tabpane lang="shell" persist=disabled >}}
+{{< tab header="Kubernetes 1.29 or newer" >}} kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/{{< param "version" >}}/visibility-apf.yaml {{< /tab >}}
+{{< tab header="Kubernetes 1.28" >}} kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/{{< param "version" >}}/visibility-apf-1-28.yaml {{< /tab >}}
+{{< /tabpane >}}
+
 ## Monitor pending workloads on demand
 
 {{< feature-state state="alpha" for_version="v0.6" >}}

test/e2e/config/1_28/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../common
+- ../../../../config/visibility-apf/1_28

test/e2e/config/kustomization.yaml → test/e2e/config/common/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ../../../config/dev
+- ../../../../config/dev
 
 replicas:
 - name: kueue-controller-manager

test/e2e/config/default/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- ../common
+- ../../../../config/visibility-apf/default