WIP: fix mode when PrioritySortingWithingCohort is disabled

kubernetes-sigs · Aug 12, 2024 · d079c97 · d079c97
1 parent e38b145
commit d079c97
Show file tree

Hide file tree

Showing 2 changed files with 128 additions and 0 deletions.
diff --git a/pkg/workload/workload.go b/pkg/workload/workload.go
@@ -540,6 +540,15 @@ func (o Ordering) GetQueueOrderTimestamp(w *kueue.Workload) *metav1.Time {
 			return &evictedCond.LastTransitionTime
 		}
 	}
+	if !features.Enabled(features.PrioritySortingWithinCohort) {
+		if evictedCond := apimeta.FindStatusCondition(w.Status.Conditions, kueue.WorkloadPreempted); evictedCond != nil &&
+			evictedCond.Status == metav1.ConditionTrue {
+			timestamp := evictedCond.LastTransitionTime.Time
+			// We add an epsilon to make sure the timestamp of the preempted
+			// workload is strictly greater that the preemptor's
+			return &metav1.Time{Time: timestamp.Add(time.Millisecond)}
+		}
+	}
 	return &w.CreationTimestamp
 }
 

diff --git a/test/integration/scheduler/preemption_test.go b/test/integration/scheduler/preemption_test.go
@@ -783,4 +783,123 @@ var _ = ginkgo.Describe("Preemption", func() {
 				testing.MakeAdmission(prodCQ.Name).Assignment(corev1.ResourceCPU, "alpha", "4").Obj())
 		})
 	})
+
+	ginkgo.Context("When borrowWithinCohort is used and PrioritySortingWithinCohort disabled", func() {
+		var (
+			aCQ, bCQ, cCQ *kueue.ClusterQueue
+			aLQ, bLQ      *kueue.LocalQueue
+			defaultFlavor *kueue.ResourceFlavor
+		)
+
+		ginkgo.BeforeEach(func() {
+			gomega.Expect(features.SetEnable(features.PrioritySortingWithinCohort, false)).To(gomega.Succeed())
+			defaultFlavor = testing.MakeResourceFlavor("default").Obj()
+			gomega.Expect(k8sClient.Create(ctx, defaultFlavor)).To(gomega.Succeed())
+
+			aCQ = testing.MakeClusterQueue("a-cq").
+				Cohort("all").
+				ResourceGroup(
+					*testing.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "0", "10").Obj(),
+				).
+				FlavorFungibility(kueue.FlavorFungibility{
+					WhenCanBorrow:  kueue.Borrow,
+					WhenCanPreempt: kueue.Preempt,
+				}).
+				Preemption(kueue.ClusterQueuePreemption{
+					ReclaimWithinCohort: kueue.PreemptionPolicyAny,
+					WithinClusterQueue:  kueue.PreemptionPolicyLowerPriority,
+				}).
+				Obj()
+			gomega.Expect(k8sClient.Create(ctx, aCQ)).To(gomega.Succeed())
+			aLQ = testing.MakeLocalQueue("a-lq", ns.Name).ClusterQueue(aCQ.Name).Obj()
+			gomega.Expect(k8sClient.Create(ctx, aLQ)).To(gomega.Succeed())
+
+			bCQ = testing.MakeClusterQueue("b-cq").
+				Cohort("all").
+				ResourceGroup(
+					*testing.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "5", "5").Obj(),
+				).
+				FlavorFungibility(kueue.FlavorFungibility{
+					WhenCanBorrow:  kueue.Borrow,
+					WhenCanPreempt: kueue.Preempt,
+				}).
+				Preemption(kueue.ClusterQueuePreemption{
+					ReclaimWithinCohort: kueue.PreemptionPolicyAny,
+					WithinClusterQueue:  kueue.PreemptionPolicyLowerPriority,
+					BorrowWithinCohort: &kueue.BorrowWithinCohort{
+						Policy:               kueue.BorrowWithinCohortPolicyLowerPriority,
+						MaxPriorityThreshold: ptr.To(veryHighPriority),
+					},
+				}).
+				Obj()
+			gomega.Expect(k8sClient.Create(ctx, bCQ)).To(gomega.Succeed())
+			bLQ = testing.MakeLocalQueue("b-lq", ns.Name).ClusterQueue(bCQ.Name).Obj()
+			gomega.Expect(k8sClient.Create(ctx, bLQ)).To(gomega.Succeed())
+
+			cCQ = testing.MakeClusterQueue("c-cq").
+				Cohort("all").
+				ResourceGroup(
+					*testing.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "5").Obj(),
+				).
+				FlavorFungibility(kueue.FlavorFungibility{
+					WhenCanBorrow:  kueue.Borrow,
+					WhenCanPreempt: kueue.Preempt,
+				}).
+				Preemption(kueue.ClusterQueuePreemption{
+					ReclaimWithinCohort: kueue.PreemptionPolicyAny,
+					WithinClusterQueue:  kueue.PreemptionPolicyLowerPriority,
+				}).
+				Obj()
+			gomega.Expect(k8sClient.Create(ctx, cCQ)).To(gomega.Succeed())
+		})
+
+		ginkgo.AfterEach(func() {
+			gomega.Expect(features.SetEnable(features.PrioritySortingWithinCohort, true)).To(gomega.Succeed())
+			gomega.Expect(util.DeleteWorkloadsInNamespace(ctx, k8sClient, ns)).To(gomega.Succeed())
+			util.ExpectObjectToBeDeleted(ctx, k8sClient, aCQ, true)
+			util.ExpectObjectToBeDeleted(ctx, k8sClient, bCQ, true)
+			util.ExpectObjectToBeDeleted(ctx, k8sClient, cCQ, true)
+		})
+
+		ginkgo.FIt("should allow preempting workloads while borrowing", func() {
+			var aWl, b1Wl, b2Wl *kueue.Workload
+
+			ginkgo.By("Create a mid priority workload in aCQ and await for admission", func() {
+				aWl = testing.MakeWorkload("a-low", ns.Name).
+					Queue(aLQ.Name).
+					Priority(midPriority).
+					Request(corev1.ResourceCPU, "4").
+					Obj()
+				gomega.Expect(k8sClient.Create(ctx, aWl)).To(gomega.Succeed())
+				util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, aWl)
+			})
+
+			ginkgo.By("Create a high priority workload b1 in bCQ and await for admission", func() {
+				b1Wl = testing.MakeWorkload("b1-high", ns.Name).
+					Queue(bLQ.Name).
+					Priority(highPriority).
+					Request(corev1.ResourceCPU, "4").
+					Obj()
+				gomega.Expect(k8sClient.Create(ctx, b1Wl)).To(gomega.Succeed())
+				util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, b1Wl)
+			})
+
+			ginkgo.By("Create a high priority workload b2 in bCQ", func() {
+				b2Wl = testing.MakeWorkload("b2-high", ns.Name).
+					Queue(bLQ.Name).
+					Priority(highPriority).
+					Request(corev1.ResourceCPU, "4").
+					Obj()
+				gomega.Expect(k8sClient.Create(ctx, b2Wl)).To(gomega.Succeed())
+			})
+
+			ginkgo.By("Await for preemption of the workload in aCQ and admission of b2", func() {
+				util.FinishEvictionForWorkloads(ctx, k8sClient, aWl)
+				util.ExpectWorkloadsToBePending(ctx, k8sClient, aWl)
+				util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, b2Wl)
+			})
+
+		})
+	})
+
 })