Create separate manifests for both FlowSchema and PriorityLevelConfig…

…uration in versions v1 and v1beta3.
kubernetes-sigs · Sep 18, 2024 · 1355dcc · 1355dcc
1 parent a7030a7
commit 1355dcc
Show file tree

Hide file tree

Showing 18 changed files with 81 additions and 15 deletions.
diff --git a/Makefile b/Makefile
@@ -266,6 +266,8 @@ artifacts: kustomize yq helm ## Generate release artifacts.
 	$(KUSTOMIZE) build config/alpha-enabled -o artifacts/manifests-alpha-enabled.yaml
 	$(KUSTOMIZE) build config/prometheus -o artifacts/prometheus.yaml
 	$(KUSTOMIZE) build config/visibility -o artifacts/visibility-api.yaml
+	$(KUSTOMIZE) build config/apf/default -o artifacts/apf.yaml
+	$(KUSTOMIZE) build config/apf/1_28 -o artifacts/apf-1-28.yaml
 	@$(call clean-manifests)
 	# Update the image tag and policy
 	$(YQ)  e  '.controllerManager.manager.image.repository = "$(IMAGE_REPO)" | .controllerManager.manager.image.tag = "$(GIT_TAG)" | .controllerManager.manager.image.pullPolicy = "IfNotPresent"' -i charts/kueue/values.yaml

diff --git a/Makefile-test.mk b/Makefile-test.mk
@@ -94,13 +94,13 @@ FORCE:
 run-test-e2e-%: K8S_VERSION = $(@:run-test-e2e-%=%)
 run-test-e2e-%: FORCE
 	@echo Running e2e for k8s ${K8S_VERSION}
-	E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) ./hack/e2e-test.sh
+	K8S_VERSION="$(K8S_VERSION)" E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) ./hack/e2e-test.sh
 	$(PROJECT_DIR)/bin/ginkgo-top -i $(ARTIFACTS)/$@/e2e.json > $(ARTIFACTS)/$@/e2e-top.yaml
 
 run-test-multikueue-e2e-%: K8S_VERSION = $(@:run-test-multikueue-e2e-%=%)
 run-test-multikueue-e2e-%: FORCE
 	@echo Running multikueue e2e for k8s ${K8S_VERSION}
-	E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) ./hack/multikueue-e2e-test.sh
+	K8S_VERSION=$(K8S_VERSION) E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) ./hack/multikueue-e2e-test.sh
 	$(PROJECT_DIR)/bin/ginkgo-top -i $(ARTIFACTS)/$@/e2e.json > $(ARTIFACTS)/$@/e2e-top.yaml
 
 SCALABILITY_RUNNER := $(PROJECT_DIR)/bin/performance-scheduler-runner

diff --git a/...ueue/templates/visibility/flowschema.yaml → .../templates/apf/visibility_flowschema.yaml b/...ueue/templates/visibility/flowschema.yaml → .../templates/apf/visibility_flowschema.yaml
@@ -1,10 +1,10 @@
-{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand") }}
-apiVersion: flowcontrol.apiserver.k8s.io/v1beta3
+apiVersion: flowcontrol.apiserver.k8s.io/{{ .Values.apf.version }}
 kind: FlowSchema
 metadata:
   labels:
   {{- include "kueue.labels" . | nindent 4 }}
   name: '{{ include "kueue.fullname" . }}-visibility'
+  namespace: '{{ .Release.Namespace }}'
 spec:
   distinguisherMethod:
     type: ByUser
@@ -29,4 +29,3 @@ spec:
         - group:
             name: system:authenticated
           kind: Group
-{{- end }}
diff --git a/...sibility/prioritylevelconfigurations.yaml → ...sibility_prioritylevelconfigurations.yaml b/...sibility/prioritylevelconfigurations.yaml → ...sibility_prioritylevelconfigurations.yaml
@@ -1,10 +1,10 @@
-{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand") }}
-apiVersion: flowcontrol.apiserver.k8s.io/v1beta3
+apiVersion: flowcontrol.apiserver.k8s.io/{{ .Values.apf.version }}
 kind: PriorityLevelConfiguration
 metadata:
   labels:
   {{- include "kueue.labels" . | nindent 4 }}
   name: '{{ include "kueue.fullname" . }}-visibility'
+  namespace: '{{ .Release.Namespace }}'
 spec:
   limited:
     lendablePercent: 0
@@ -16,4 +16,3 @@ spec:
       type: Queue
     nominalConcurrencyShares: 10
   type: Limited
-{{- end }}
diff --git a/charts/kueue/values.yaml b/charts/kueue/values.yaml
@@ -7,6 +7,9 @@ fullnameOverride: ""
 enablePrometheus: false
 # Enable x509 automated certificate management using cert-manager (cert-manager.io)
 enableCertManager: false
+# Customize APF
+apf:
+  version: v1
 # Customize controllerManager
 controllerManager:
   #featureGates:

diff --git a/config/apf/1_28/kustomization.yaml b/config/apf/1_28/kustomization.yaml
@@ -0,0 +1,19 @@
+# This overlay builds the APF components for Kubernetes version 1.28 to be used in combination
+# with other overlays.
+
+resources:
+- ../default/
+
+patches:
+  - target:
+      kind: FlowSchema
+    patch: |
+      - op: replace
+        path: /apiVersion
+        value: flowcontrol.apiserver.k8s.io/v1beta3
+  - target:
+      kind: PriorityLevelConfiguration
+    patch: |
+      - op: replace
+        path: /apiVersion
+        value: flowcontrol.apiserver.k8s.io/v1beta3
diff --git a/config/apf/default/kustomization.yaml b/config/apf/default/kustomization.yaml
@@ -0,0 +1,17 @@
+# This overlay builds the APF components to be used in combination
+# with other overlays.
+
+namespace: kueue-system
+namePrefix: kueue-
+
+commonLabels:
+  control-plane: controller-manager
+
+labels:
+  - pairs:
+      app.kubernetes.io/name: kueue
+      app.kubernetes.io/component: controller
+    includeTemplates: true
+
+resources:
+  - ../../components/apf/
diff --git a/config/components/apf/kustomization.yaml b/config/components/apf/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+- visibility_flowschema.yaml
+- visibility_prioritylevelconfigurations.yaml
diff --git a/config/components/visibility/flowschema.yaml → ...components/apf/visibility_flowschema.yaml b/config/components/visibility/flowschema.yaml → ...components/apf/visibility_flowschema.yaml
@@ -1,7 +1,8 @@
-apiVersion: flowcontrol.apiserver.k8s.io/v1beta3
+apiVersion: flowcontrol.apiserver.k8s.io/v1
 kind: FlowSchema
 metadata:
   name: visibility
+  namespace: system
 spec:
   distinguisherMethod:
     type: ByUser

diff --git a/...sibility/prioritylevelconfigurations.yaml → ...sibility_prioritylevelconfigurations.yaml b/...sibility/prioritylevelconfigurations.yaml → ...sibility_prioritylevelconfigurations.yaml
@@ -1,7 +1,8 @@
-apiVersion: flowcontrol.apiserver.k8s.io/v1beta3
+apiVersion: flowcontrol.apiserver.k8s.io/v1
 kind: PriorityLevelConfiguration
 metadata:
   name: visibility
+  namespace: system
 spec:
   limited:
     lendablePercent: 0

diff --git a/config/components/visibility/kustomization.yaml b/config/components/visibility/kustomization.yaml
@@ -2,5 +2,3 @@ resources:
 - apiservice.yaml
 - role_binding.yaml
 - service.yaml
-- flowschema.yaml
-- prioritylevelconfigurations.yaml
diff --git a/hack/e2e-common.sh b/hack/e2e-common.sh
@@ -75,7 +75,13 @@ function cluster_kind_load_image {
 # $1 cluster
 function cluster_kueue_deploy {
     kubectl config use-context "kind-${1}"
-    kubectl apply --server-side -k test/e2e/config
+    if [ "${K8S_VERSION%.*}" = "1.28" ]; then
+        echo "Applying 1.28"
+        kubectl apply --server-side -k test/e2e/config/1_28
+    else
+        echo "Applying Default"
+        kubectl apply --server-side -k test/e2e/config/default
+    fi
 }
 
 #$1 - cluster name

diff --git a/hack/update-helm.sh b/hack/update-helm.sh
@@ -23,11 +23,13 @@ SRC_CRD_DIR=config/components/crd/bases
 SRC_RBAC_DIR=config/components/rbac
 SRC_WEBHOOK_DIR=config/components/webhook
 SRC_VISIBILITY_DIR=config/components/visibility
+SRC_APF_DIR=config/components/apf
 
 DEST_CRD_DIR=charts/kueue/templates/crd
 DEST_RBAC_DIR=charts/kueue/templates/rbac
 DEST_WEBHOOK_DIR=charts/kueue/templates/webhook
 DEST_VISIBILITY_DIR=charts/kueue/templates/visibility
+DEST_APF_DIR=charts/kueue/templates/apf
 
 YQ=./bin/yq
 SED=${SED:-/usr/bin/sed}
@@ -48,6 +50,8 @@ find $SRC_RBAC_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_RBAC_D
 find $SRC_WEBHOOK_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_WEBHOOK_DIR \;
 # shellcheck disable=SC2086
 find $SRC_VISIBILITY_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_VISIBILITY_DIR \;
+# shellcheck disable=SC2086
+find $SRC_APF_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_APF_DIR \;
 $YQ -N -s '.kind' ${DEST_WEBHOOK_DIR}/manifests.yaml
 rm ${DEST_WEBHOOK_DIR}/manifests.yaml
 files=("MutatingWebhookConfiguration.yml" "ValidatingWebhookConfiguration.yml")
@@ -305,3 +309,11 @@ EOT
   } > "${output_file}.tmp"
   mv "${output_file}.tmp" "${output_file}"
 done
+
+# Replace flowcontrol version on APF directory
+for output_file in "${DEST_APF_DIR}"/*.yaml; do
+  $YQ -N -i '.apiVersion = "flowcontrol.apiserver.k8s.io/{{ .Values.apf.version }}"' "$output_file"
+  $YQ -N -i '.metadata.name |= "{{ include \"kueue.fullname\" . }}-" + .' "$output_file"
+  $YQ -N -i '.metadata.namespace = "{{ .Release.Namespace }}"' "$output_file"
+  $SED -i '/^metadata:.*/a\  labels:\n  {{- include "kueue.labels" . | nindent 4 }}' "$output_file"
+done
diff --git a/test/e2e/config/1_28/kustomization.yaml b/test/e2e/config/1_28/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+- ../common
+- ../../../../config/apf/1_28
diff --git a/...e2e/config/controller_manager_config.yaml → ...fig/common/controller_manager_config.yaml b/...e2e/config/controller_manager_config.yaml → ...fig/common/controller_manager_config.yaml
diff --git a/test/e2e/config/kustomization.yaml → test/e2e/config/common/kustomization.yaml b/test/e2e/config/kustomization.yaml → test/e2e/config/common/kustomization.yaml
@@ -2,8 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- ../../../config/dev
-- ../../../config/visibility
+- ../../../../config/dev
+- ../../../../config/visibility
 
 replicas:
 - name: kueue-controller-manager

diff --git a/test/e2e/config/manager_e2e_patch.yaml → .../e2e/config/common/manager_e2e_patch.yaml b/test/e2e/config/manager_e2e_patch.yaml → .../e2e/config/common/manager_e2e_patch.yaml
diff --git a/test/e2e/config/default/kustomization.yaml b/test/e2e/config/default/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+- ../common
+- ../../../../config/apf/default