From 39e933360a3d49a492e7561ffbe3394cdd56d2f1 Mon Sep 17 00:00:00 2001 From: Alex Wang Date: Thu, 17 Mar 2022 12:59:58 +0800 Subject: [PATCH] add integration test for workload priority Signed-off-by: Alex Wang --- .../controller/job/job_controller_test.go | 27 ++++++---- test/integration/scheduler/scheduler_test.go | 52 +++++++++++++++++++ 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/test/integration/controller/job/job_controller_test.go b/test/integration/controller/job/job_controller_test.go index bd1eeba8fd..81bd60b363 100644 --- a/test/integration/controller/job/job_controller_test.go +++ b/test/integration/controller/job/job_controller_test.go @@ -37,13 +37,15 @@ import ( ) const ( - parallelism = 4 - jobName = "test-job" - jobNamespace = "default" - jobKey = jobNamespace + "/" + jobName - labelKey = "cloud.provider.com/instance" - flavorOnDemand = "on-demand" - flavorSpot = "spot" + parallelism = 4 + jobName = "test-job" + jobNamespace = "default" + jobKey = jobNamespace + "/" + jobName + labelKey = "cloud.provider.com/instance" + flavorOnDemand = "on-demand" + flavorSpot = "spot" + priorityClassName = "test-priority-class" + priorityValue = 10 ) // +kubebuilder:docs-gen:collapse=Imports @@ -51,7 +53,10 @@ const ( var _ = ginkgo.Describe("Job controller", func() { ginkgo.It("Should reconcile workload and job", func() { ginkgo.By("checking the job gets suspended when created unsuspended") - job := testing.MakeJob(jobName, jobNamespace).Obj() + priorityClass := testing.MakePriorityClass(priorityClassName). + PriorityValue(int32(priorityValue)).Obj() + gomega.Expect(k8sClient.Create(ctx, priorityClass)).Should(gomega.Succeed()) + job := testing.MakeJob(jobName, jobNamespace).PriorityClass(priorityClassName).Obj() gomega.Expect(k8sClient.Create(ctx, job)).Should(gomega.Succeed()) lookupKey := types.NamespacedName{Name: jobName, Namespace: jobNamespace} createdJob := &batchv1.Job{} @@ -70,6 +75,10 @@ var _ = ginkgo.Describe("Job controller", func() { }, framework.Timeout, framework.Interval).Should(gomega.BeTrue()) gomega.Expect(createdWorkload.Spec.QueueName).Should(gomega.Equal("")) + ginkgo.By("checking the workload is created with priority and priorityName") + gomega.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(priorityClassName)) + gomega.Expect(*createdWorkload.Spec.Priority).Should(gomega.Equal(int32(priorityValue))) + ginkgo.By("checking the workload is updated with queue name when the job does") jobQueueName := "test-queue" createdJob.Annotations = map[string]string{constants.QueueAnnotation: jobQueueName} @@ -82,7 +91,7 @@ var _ = ginkgo.Describe("Job controller", func() { }, framework.Timeout, framework.Interval).Should(gomega.BeTrue()) ginkgo.By("checking a second non-matching workload is deleted") - secondWl, _ := workloadjob.ConstructWorkloadFor(createdJob, scheme.Scheme) + secondWl, _ := workloadjob.ConstructWorkloadFor(ctx, k8sClient, createdJob, scheme.Scheme) secondWl.Name = "second-workload" secondWl.Spec.PodSets[0].Count = parallelism + 1 gomega.Expect(k8sClient.Create(ctx, secondWl)).Should(gomega.Succeed()) diff --git a/test/integration/scheduler/scheduler_test.go b/test/integration/scheduler/scheduler_test.go index 50e613e9ba..fa5ead8dce 100644 --- a/test/integration/scheduler/scheduler_test.go +++ b/test/integration/scheduler/scheduler_test.go @@ -269,4 +269,56 @@ var _ = ginkgo.Describe("Scheduler", func() { return createdJob.Spec.Suspend }, framework.Timeout, framework.Interval).Should(gomega.Equal(pointer.Bool(false))) }) + + ginkgo.It("Should schedule jobs according to their priorities", func() { + queue := testing.MakeQueue("queue", ns.Name).ClusterQueue(prodClusterQ.Name).Obj() + + highPriorityClass := testing.MakePriorityClass("high-priority-class").PriorityValue(100).Obj() + gomega.Expect(k8sClient.Create(ctx, highPriorityClass)).Should(gomega.Succeed()) + + lowPriorityClass := testing.MakePriorityClass("low-priority-class").PriorityValue(10).Obj() + gomega.Expect(k8sClient.Create(ctx, lowPriorityClass)).Should(gomega.Succeed()) + + jobLowPriority := testing.MakeJob("job-low-priority", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "5").PriorityClass(lowPriorityClass.Name).Obj() + gomega.Expect(k8sClient.Create(ctx, jobLowPriority)).Should(gomega.Succeed()) + jobHighPriority := testing.MakeJob("job-high-priority", ns.Name).Queue(queue.Name).Request(corev1.ResourceCPU, "5").PriorityClass(highPriorityClass.Name).Obj() + gomega.Expect(k8sClient.Create(ctx, jobHighPriority)).Should(gomega.Succeed()) + + ginkgo.By("checking that workload1 is created with priority and priorityName") + createdLowPriorityWorkload := &kueue.QueuedWorkload{} + gomega.Eventually(func() bool { + lookupKey := types.NamespacedName{Name: jobLowPriority.Name, Namespace: jobLowPriority.Namespace} + err := k8sClient.Get(ctx, lookupKey, createdLowPriorityWorkload) + return err == nil + }, framework.Timeout, framework.Interval).Should(gomega.BeTrue()) + gomega.Expect(createdLowPriorityWorkload.Spec.PriorityClassName).Should(gomega.Equal(lowPriorityClass.Name)) + gomega.Expect(*createdLowPriorityWorkload.Spec.Priority).Should(gomega.Equal(lowPriorityClass.Value)) + + ginkgo.By("checking that workload2 is created with priority and priorityName") + createdHighPriorityWorkload := &kueue.QueuedWorkload{} + gomega.Eventually(func() bool { + lookupKey := types.NamespacedName{Name: jobHighPriority.Name, Namespace: jobHighPriority.Namespace} + err := k8sClient.Get(ctx, lookupKey, createdHighPriorityWorkload) + return err == nil + }, framework.Timeout, framework.Interval).Should(gomega.BeTrue()) + gomega.Expect(createdHighPriorityWorkload.Spec.PriorityClassName).Should(gomega.Equal(highPriorityClass.Name)) + gomega.Expect(*createdHighPriorityWorkload.Spec.Priority).Should(gomega.Equal(highPriorityClass.Value)) + + // delay creating the queue until after workloads are created. + gomega.Expect(k8sClient.Create(ctx, queue)).Should(gomega.Succeed()) + + ginkgo.By("checking the job with low priority continues to be suspended") + createdJob1 := &batchv1.Job{} + gomega.Consistently(func() bool { + return k8sClient.Get(ctx, types.NamespacedName{Name: jobLowPriority.Name, Namespace: ns.Name}, + createdJob1) == nil && *createdJob1.Spec.Suspend + }, framework.ConsistentDuration, framework.Interval).Should(gomega.BeTrue()) + + ginkgo.By("checking the job with high priority starts") + createdJob2 := &batchv1.Job{} + gomega.Eventually(func() *bool { + gomega.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: jobHighPriority.Name, Namespace: ns.Name}, createdJob2)).Should(gomega.Succeed()) + return createdJob2.Spec.Suspend + }, framework.Timeout, framework.Interval).Should(gomega.Equal(pointer.Bool(false))) + }) })