diff --git a/Makefile b/Makefile index bbf7b090d52..ac4ca3f4f4f 100644 --- a/Makefile +++ b/Makefile @@ -266,6 +266,8 @@ artifacts: kustomize yq helm ## Generate release artifacts. $(KUSTOMIZE) build config/alpha-enabled -o artifacts/manifests-alpha-enabled.yaml $(KUSTOMIZE) build config/prometheus -o artifacts/prometheus.yaml $(KUSTOMIZE) build config/visibility -o artifacts/visibility-api.yaml + $(KUSTOMIZE) build config/apf/default -o artifacts/apf.yaml + $(KUSTOMIZE) build config/apf/1_28 -o artifacts/apf-1-28.yaml @$(call clean-manifests) # Update the image tag and policy $(YQ) e '.controllerManager.manager.image.repository = "$(IMAGE_REPO)" | .controllerManager.manager.image.tag = "$(GIT_TAG)" | .controllerManager.manager.image.pullPolicy = "IfNotPresent"' -i charts/kueue/values.yaml diff --git a/Makefile-test.mk b/Makefile-test.mk index 6bc17584b26..77c76c66100 100644 --- a/Makefile-test.mk +++ b/Makefile-test.mk @@ -94,13 +94,13 @@ FORCE: run-test-e2e-%: K8S_VERSION = $(@:run-test-e2e-%=%) run-test-e2e-%: FORCE @echo Running e2e for k8s ${K8S_VERSION} - E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) ./hack/e2e-test.sh + K8S_VERSION="$(K8S_VERSION)" E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) ./hack/e2e-test.sh $(PROJECT_DIR)/bin/ginkgo-top -i $(ARTIFACTS)/$@/e2e.json > $(ARTIFACTS)/$@/e2e-top.yaml run-test-multikueue-e2e-%: K8S_VERSION = $(@:run-test-multikueue-e2e-%=%) run-test-multikueue-e2e-%: FORCE @echo Running multikueue e2e for k8s ${K8S_VERSION} - E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) ./hack/multikueue-e2e-test.sh + K8S_VERSION=$(K8S_VERSION) E2E_KIND_VERSION="kindest/node:v$(K8S_VERSION)" KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" IMAGE_TAG=$(IMAGE_TAG) GINKGO_ARGS="$(GINKGO_ARGS)" JOBSET_VERSION=$(JOBSET_VERSION) KUBEFLOW_VERSION=$(KUBEFLOW_VERSION) KUBEFLOW_MPI_VERSION=$(KUBEFLOW_MPI_VERSION) ./hack/multikueue-e2e-test.sh $(PROJECT_DIR)/bin/ginkgo-top -i $(ARTIFACTS)/$@/e2e.json > $(ARTIFACTS)/$@/e2e-top.yaml SCALABILITY_RUNNER := $(PROJECT_DIR)/bin/performance-scheduler-runner diff --git a/charts/kueue/templates/visibility/flowschema.yaml b/charts/kueue/templates/apf/visibility_flowschema.yaml similarity index 78% rename from charts/kueue/templates/visibility/flowschema.yaml rename to charts/kueue/templates/apf/visibility_flowschema.yaml index af42e071181..e6544a54bf5 100644 --- a/charts/kueue/templates/visibility/flowschema.yaml +++ b/charts/kueue/templates/apf/visibility_flowschema.yaml @@ -1,10 +1,10 @@ -{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand") }} -apiVersion: flowcontrol.apiserver.k8s.io/v1beta3 +apiVersion: flowcontrol.apiserver.k8s.io/{{ .Values.apf.version }} kind: FlowSchema metadata: labels: {{- include "kueue.labels" . | nindent 4 }} name: '{{ include "kueue.fullname" . }}-visibility' + namespace: '{{ .Release.Namespace }}' spec: distinguisherMethod: type: ByUser @@ -29,4 +29,3 @@ spec: - group: name: system:authenticated kind: Group -{{- end }} diff --git a/charts/kueue/templates/visibility/prioritylevelconfigurations.yaml b/charts/kueue/templates/apf/visibility_prioritylevelconfigurations.yaml similarity index 66% rename from charts/kueue/templates/visibility/prioritylevelconfigurations.yaml rename to charts/kueue/templates/apf/visibility_prioritylevelconfigurations.yaml index 8a4ab6a2aeb..4767944077b 100644 --- a/charts/kueue/templates/visibility/prioritylevelconfigurations.yaml +++ b/charts/kueue/templates/apf/visibility_prioritylevelconfigurations.yaml @@ -1,10 +1,10 @@ -{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand") }} -apiVersion: flowcontrol.apiserver.k8s.io/v1beta3 +apiVersion: flowcontrol.apiserver.k8s.io/{{ .Values.apf.version }} kind: PriorityLevelConfiguration metadata: labels: {{- include "kueue.labels" . | nindent 4 }} name: '{{ include "kueue.fullname" . }}-visibility' + namespace: '{{ .Release.Namespace }}' spec: limited: lendablePercent: 0 @@ -16,4 +16,3 @@ spec: type: Queue nominalConcurrencyShares: 10 type: Limited -{{- end }} diff --git a/charts/kueue/values.yaml b/charts/kueue/values.yaml index 0fa2ef3f9e3..55bf93637bf 100644 --- a/charts/kueue/values.yaml +++ b/charts/kueue/values.yaml @@ -7,6 +7,9 @@ fullnameOverride: "" enablePrometheus: false # Enable x509 automated certificate management using cert-manager (cert-manager.io) enableCertManager: false +# Customize APF +apf: + version: v1 # Customize controllerManager controllerManager: #featureGates: diff --git a/config/apf/1_28/kustomization.yaml b/config/apf/1_28/kustomization.yaml new file mode 100644 index 00000000000..23832e5e175 --- /dev/null +++ b/config/apf/1_28/kustomization.yaml @@ -0,0 +1,19 @@ +# This overlay builds the APF components for Kubernetes version 1.28 to be used in combination +# with other overlays. + +resources: +- ../default/ + +patches: + - target: + kind: FlowSchema + patch: | + - op: replace + path: /apiVersion + value: flowcontrol.apiserver.k8s.io/v1beta3 + - target: + kind: PriorityLevelConfiguration + patch: | + - op: replace + path: /apiVersion + value: flowcontrol.apiserver.k8s.io/v1beta3 \ No newline at end of file diff --git a/config/apf/default/kustomization.yaml b/config/apf/default/kustomization.yaml new file mode 100644 index 00000000000..309342fc82d --- /dev/null +++ b/config/apf/default/kustomization.yaml @@ -0,0 +1,17 @@ +# This overlay builds the APF components to be used in combination +# with other overlays. + +namespace: kueue-system +namePrefix: kueue- + +commonLabels: + control-plane: controller-manager + +labels: + - pairs: + app.kubernetes.io/name: kueue + app.kubernetes.io/component: controller + includeTemplates: true + +resources: + - ../../components/apf/ \ No newline at end of file diff --git a/config/components/apf/kustomization.yaml b/config/components/apf/kustomization.yaml new file mode 100644 index 00000000000..88406a81758 --- /dev/null +++ b/config/components/apf/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- visibility_flowschema.yaml +- visibility_prioritylevelconfigurations.yaml diff --git a/config/components/visibility/flowschema.yaml b/config/components/apf/visibility_flowschema.yaml similarity index 90% rename from config/components/visibility/flowschema.yaml rename to config/components/apf/visibility_flowschema.yaml index 1c67a5bd4ac..091cb310595 100644 --- a/config/components/visibility/flowschema.yaml +++ b/config/components/apf/visibility_flowschema.yaml @@ -1,7 +1,8 @@ -apiVersion: flowcontrol.apiserver.k8s.io/v1beta3 +apiVersion: flowcontrol.apiserver.k8s.io/v1 kind: FlowSchema metadata: name: visibility + namespace: system spec: distinguisherMethod: type: ByUser diff --git a/config/components/visibility/prioritylevelconfigurations.yaml b/config/components/apf/visibility_prioritylevelconfigurations.yaml similarity index 80% rename from config/components/visibility/prioritylevelconfigurations.yaml rename to config/components/apf/visibility_prioritylevelconfigurations.yaml index e9112bd4147..c3338a79503 100644 --- a/config/components/visibility/prioritylevelconfigurations.yaml +++ b/config/components/apf/visibility_prioritylevelconfigurations.yaml @@ -1,7 +1,8 @@ -apiVersion: flowcontrol.apiserver.k8s.io/v1beta3 +apiVersion: flowcontrol.apiserver.k8s.io/v1 kind: PriorityLevelConfiguration metadata: name: visibility + namespace: system spec: limited: lendablePercent: 0 diff --git a/config/components/visibility/kustomization.yaml b/config/components/visibility/kustomization.yaml index 25f54e445d8..ad7f432243d 100644 --- a/config/components/visibility/kustomization.yaml +++ b/config/components/visibility/kustomization.yaml @@ -2,5 +2,3 @@ resources: - apiservice.yaml - role_binding.yaml - service.yaml -- flowschema.yaml -- prioritylevelconfigurations.yaml diff --git a/hack/e2e-common.sh b/hack/e2e-common.sh index d28a4476912..1f1e5cab2a2 100644 --- a/hack/e2e-common.sh +++ b/hack/e2e-common.sh @@ -75,7 +75,13 @@ function cluster_kind_load_image { # $1 cluster function cluster_kueue_deploy { kubectl config use-context "kind-${1}" - kubectl apply --server-side -k test/e2e/config + if [ "${K8S_VERSION%.*}" = "1.28" ]; then + echo "Applying 1.28" + kubectl apply --server-side -k test/e2e/config/1_28 + else + echo "Applying Default" + kubectl apply --server-side -k test/e2e/config/default + fi } #$1 - cluster name diff --git a/hack/update-helm.sh b/hack/update-helm.sh index 1a3a3b749e2..375a047e41b 100755 --- a/hack/update-helm.sh +++ b/hack/update-helm.sh @@ -23,11 +23,13 @@ SRC_CRD_DIR=config/components/crd/bases SRC_RBAC_DIR=config/components/rbac SRC_WEBHOOK_DIR=config/components/webhook SRC_VISIBILITY_DIR=config/components/visibility +SRC_APF_DIR=config/components/apf DEST_CRD_DIR=charts/kueue/templates/crd DEST_RBAC_DIR=charts/kueue/templates/rbac DEST_WEBHOOK_DIR=charts/kueue/templates/webhook DEST_VISIBILITY_DIR=charts/kueue/templates/visibility +DEST_APF_DIR=charts/kueue/templates/apf YQ=./bin/yq SED=${SED:-/usr/bin/sed} @@ -48,6 +50,8 @@ find $SRC_RBAC_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_RBAC_D find $SRC_WEBHOOK_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_WEBHOOK_DIR \; # shellcheck disable=SC2086 find $SRC_VISIBILITY_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_VISIBILITY_DIR \; +# shellcheck disable=SC2086 +find $SRC_APF_DIR -name "*.yaml" $EXCLUDE_FILES_ARGS -exec cp "{}" $DEST_APF_DIR \; $YQ -N -s '.kind' ${DEST_WEBHOOK_DIR}/manifests.yaml rm ${DEST_WEBHOOK_DIR}/manifests.yaml files=("MutatingWebhookConfiguration.yml" "ValidatingWebhookConfiguration.yml") @@ -305,3 +309,11 @@ EOT } > "${output_file}.tmp" mv "${output_file}.tmp" "${output_file}" done + +# Replace flowcontrol version on APF directory +for output_file in "${DEST_APF_DIR}"/*.yaml; do + $YQ -N -i '.apiVersion = "flowcontrol.apiserver.k8s.io/{{ .Values.apf.version }}"' "$output_file" + $YQ -N -i '.metadata.name |= "{{ include \"kueue.fullname\" . }}-" + .' "$output_file" + $YQ -N -i '.metadata.namespace = "{{ .Release.Namespace }}"' "$output_file" + $SED -i '/^metadata:.*/a\ labels:\n {{- include "kueue.labels" . | nindent 4 }}' "$output_file" +done \ No newline at end of file diff --git a/test/e2e/config/1_28/kustomization.yaml b/test/e2e/config/1_28/kustomization.yaml new file mode 100644 index 00000000000..73bb6003c45 --- /dev/null +++ b/test/e2e/config/1_28/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ../common +- ../../../../config/apf/1_28 \ No newline at end of file diff --git a/test/e2e/config/controller_manager_config.yaml b/test/e2e/config/common/controller_manager_config.yaml similarity index 100% rename from test/e2e/config/controller_manager_config.yaml rename to test/e2e/config/common/controller_manager_config.yaml diff --git a/test/e2e/config/kustomization.yaml b/test/e2e/config/common/kustomization.yaml similarity index 88% rename from test/e2e/config/kustomization.yaml rename to test/e2e/config/common/kustomization.yaml index 064f203d8b6..1a1b18f6d9c 100644 --- a/test/e2e/config/kustomization.yaml +++ b/test/e2e/config/common/kustomization.yaml @@ -2,8 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- ../../../config/dev -- ../../../config/visibility +- ../../../../config/dev +- ../../../../config/visibility replicas: - name: kueue-controller-manager diff --git a/test/e2e/config/manager_e2e_patch.yaml b/test/e2e/config/common/manager_e2e_patch.yaml similarity index 100% rename from test/e2e/config/manager_e2e_patch.yaml rename to test/e2e/config/common/manager_e2e_patch.yaml diff --git a/test/e2e/config/default/kustomization.yaml b/test/e2e/config/default/kustomization.yaml new file mode 100644 index 00000000000..e4399391d24 --- /dev/null +++ b/test/e2e/config/default/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ../common +- ../../../../config/apf/default \ No newline at end of file