From dad5559abe9d24d2190749e580c32ba681cd19e5 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 10 Apr 2024 15:45:37 -0400 Subject: [PATCH] Revert "drm/sched: Convert the GPU scheduler to variable number ..." Revert submission 1033322 Reason for revert: Breaks GPU reset Reverted Changes: I518055d36:drm/amdgpu: drop setting buffer funcs in sdma442 I3161b3ba3:drm/amdgpu: fix buffer funcs setting order on susp... If96a48811:drm/amdklc: fix a intree build error. I370135be6:drm/sched: Don't disturb the entity when in RR-mod... I051ca6e0b:drm/sched: Move free worker re-queuing out of the ... I25fb2785c:drm/sched: Rename drm_sched_get_cleanup_job to be ... I666125051:drm/sched: Rename drm_sched_run_job_queue_if_ready... I9a32472a9:drm/sched: Rename drm_sched_free_job_queue to be m... I4c2c9bed1:drm/sched: Re-queue run job worker when drm_sched_... Ie513a85c7:drm/sched: Drain all entities in DRM sched run job... I855c12a4e:drm/sched: Split free_job into own work item I7a116284b:drm/sched: Convert drm scheduler to use a work que... Iffc1aa4cd:drm/amdgpu: move buffer funcs setting up a level I3753e9dd4:drm/sched: Convert the GPU scheduler to variable n... Change-Id: I8fa3facd7111e26a08ff77e6345b4437631e9ac0 --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 4 +- drivers/gpu/drm/etnaviv/etnaviv_sched.c | 1 - drivers/gpu/drm/lima/lima_sched.c | 1 - drivers/gpu/drm/msm/msm_ringbuffer.c | 1 - drivers/gpu/drm/panfrost/panfrost_job.c | 1 - drivers/gpu/drm/scheduler/sched_entity.c | 18 +---- drivers/gpu/drm/scheduler/sched_main.c | 93 ++++++---------------- drivers/gpu/drm/v3d/v3d_sched.c | 5 -- include/drm/gpu_scheduler.h | 6 +- 10 files changed, 32 insertions(+), 99 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 76433c4ed704c..1870c7e20e62c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -2661,7 +2661,6 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) } r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, - DRM_SCHED_PRIORITY_COUNT, ring->num_hw_submission, 0, timeout, adev->reset_domain->wq, ring->sched_score, ring->name, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 11ff785491965..bbaff3efc9473 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -326,8 +326,8 @@ void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched) int i; /* Signal all jobs not yet scheduled */ - for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { - struct drm_sched_rq *rq = sched->sched_rq[i]; + for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_KERNEL; i--) { + struct drm_sched_rq *rq = &sched->sched_rq[i]; spin_lock(&rq->lock); list_for_each_entry(s_entity, &rq->entities, list) { while ((s_job = to_drm_sched_job(spsc_queue_pop(&s_entity->job_queue)))) { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 9b79f218e21af..345fec6cb1a4c 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -135,7 +135,6 @@ int etnaviv_sched_init(struct etnaviv_gpu *gpu) int ret; ret = drm_sched_init(&gpu->sched, &etnaviv_sched_ops, - DRM_SCHED_PRIORITY_COUNT, etnaviv_hw_jobs_limit, etnaviv_job_hang_limit, msecs_to_jiffies(500), NULL, NULL, dev_name(gpu->dev), gpu->dev); diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c index 0febe8ceb43b8..9bce6375c5d91 100644 --- a/drivers/gpu/drm/lima/lima_sched.c +++ b/drivers/gpu/drm/lima/lima_sched.c @@ -489,7 +489,6 @@ int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name) INIT_WORK(&pipe->recover_work, lima_sched_recover_work); return drm_sched_init(&pipe->base, &lima_sched_ops, - DRM_SCHED_PRIORITY_COUNT, 1, lima_job_hang_limit, msecs_to_jiffies(timeout), NULL, diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c index 95257ab0185dc..1651de918e878 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.c +++ b/drivers/gpu/drm/msm/msm_ringbuffer.c @@ -95,7 +95,6 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id, sched_timeout = MAX_SCHEDULE_TIMEOUT; ret = drm_sched_init(&ring->sched, &msm_sched_ops, - DRM_SCHED_PRIORITY_COUNT, num_hw_submissions, 0, sched_timeout, NULL, NULL, to_msm_bo(ring->bo)->name, gpu->dev->dev); if (ret) { diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c index 6795c6d4e615b..ebd14e3d8bef6 100644 --- a/drivers/gpu/drm/panfrost/panfrost_job.c +++ b/drivers/gpu/drm/panfrost/panfrost_job.c @@ -871,7 +871,6 @@ int panfrost_job_init(struct panfrost_device *pfdev) ret = drm_sched_init(&js->queue[j].sched, &panfrost_sched_ops, - DRM_SCHED_PRIORITY_COUNT, nentries, 0, msecs_to_jiffies(JOB_TIMEOUT_MS), pfdev->reset.wq, diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index 210929367a8bc..4b1e2b8306205 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -75,20 +75,8 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, RCU_INIT_POINTER(entity->last_scheduled, NULL); RB_CLEAR_NODE(&entity->rb_tree_node); - if (!sched_list[0]->sched_rq) { - /* Warn drivers not to do this and to fix their DRM - * calling order. - */ - pr_warn("%s: called with uninitialized scheduler\n", __func__); - } else if (num_sched_list) { - /* The "priority" of an entity cannot exceed the number - * of run-queues of a scheduler. - */ - if (entity->priority >= sched_list[0]->num_rqs) - entity->priority = max_t(u32, sched_list[0]->num_rqs, - DRM_SCHED_PRIORITY_KERNEL); - entity->rq = sched_list[0]->sched_rq[entity->priority]; - } + if(num_sched_list) + entity->rq = &sched_list[0]->sched_rq[entity->priority]; init_completion(&entity->entity_idle); @@ -646,7 +634,7 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity) spin_lock(&entity->rq_lock); sched = drm_sched_pick_best(entity->sched_list, entity->num_sched_list); - rq = sched ? sched->sched_rq[entity->priority] : NULL; + rq = sched ? &sched->sched_rq[entity->priority] : NULL; if (rq != entity->rq) { drm_sched_rq_remove_entity(entity->rq, entity); entity->rq = rq; diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index a828a4d70bb8a..000d7b6664960 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -732,14 +732,8 @@ int drm_sched_job_init(struct drm_sched_job *job, struct drm_sched_entity *entity, u32 credits, void *owner) { - if (!entity->rq) { - /* This will most likely be followed by missing frames - * or worse--a blank screen--leave a trail in the - * logs, so this can be debugged easier. - */ - drm_err(job->sched, "%s: entity has no rq!\n", __func__); + if (!entity->rq) return -ENOENT; - } if (unlikely(!credits)) { pr_err("*ERROR* %s: credits cannot be 0!\n", __func__); @@ -784,7 +778,7 @@ void drm_sched_job_arm(struct drm_sched_job *job) sched = entity->rq->sched; job->sched = sched; - job->s_priority = entity->priority; + job->s_priority = entity->rq - sched->sched_rq; job->id = atomic64_inc_return(&sched->job_id_count); drm_sched_fence_init(job->s_fence, job->entity); @@ -999,11 +993,10 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) int i; /* Kernel run queue has higher priority than normal run queue*/ - for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { - entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ? - drm_sched_rq_select_entity_fifo(sched, sched->sched_rq[i]) : - drm_sched_rq_select_entity_rr(sched, sched->sched_rq[i]); - + for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_KERNEL; i--) { + entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ? + drm_sched_rq_select_entity_fifo(sched, &sched->sched_rq[i]) : + drm_sched_rq_select_entity_rr(sched, &sched->sched_rq[i]); if (entity) break; } @@ -1183,9 +1176,7 @@ static int drm_sched_main(void *param) * * @sched: scheduler instance * @ops: backend operations for this scheduler - * @num_rqs: number of runqueues, one for each priority, up to DRM_SCHED_PRIORITY_COUNT * @credit_limit: the number of credits this scheduler can hold from all jobs - * @num_rqs: number of runqueues, one for each priority, up to DRM_SCHED_PRIORITY_COUNT * @hang_limit: number of times to allow a job to hang before dropping it * @timeout: timeout value in jiffies for the scheduler * @timeout_wq: workqueue to use for timeout work. If NULL, the system_wq is @@ -1198,12 +1189,11 @@ static int drm_sched_main(void *param) */ int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_backend_ops *ops, - u32 num_rqs, u32 credit_limit, unsigned int hang_limit, + u32 credit_limit, unsigned int hang_limit, long timeout, struct workqueue_struct *timeout_wq, atomic_t *score, const char *name, struct device *dev) { int i, ret; - sched->ops = ops; sched->credit_limit = credit_limit; sched->name = name; @@ -1213,35 +1203,8 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, sched->score = score ? score : &sched->_score; sched->dev = dev; - if (num_rqs > DRM_SCHED_PRIORITY_COUNT) { - /* This is a gross violation--tell drivers what the problem is. - */ - drm_err(sched, "%s: num_rqs cannot be greater than DRM_SCHED_PRIORITY_COUNT\n", - __func__); - return -EINVAL; - } else if (sched->sched_rq) { - /* Not an error, but warn anyway so drivers can - * fine-tune their DRM calling order, and return all - * is good. - */ - drm_warn(sched, "%s: scheduler already initialized!\n", __func__); - return 0; - } - - sched->sched_rq = kmalloc_array(num_rqs, sizeof(*sched->sched_rq), - GFP_KERNEL | __GFP_ZERO); - if (!sched->sched_rq) { - drm_err(sched, "%s: out of memory for sched_rq\n", __func__); - return -ENOMEM; - } - sched->num_rqs = num_rqs; - ret = -ENOMEM; - for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { - sched->sched_rq[i] = kzalloc(sizeof(*sched->sched_rq[i]), GFP_KERNEL); - if (!sched->sched_rq[i]) - goto Out_unroll; - drm_sched_rq_init(sched, sched->sched_rq[i]); - } + for (i = DRM_SCHED_PRIORITY_KERNEL; i < DRM_SCHED_PRIORITY_COUNT; i++) + drm_sched_rq_init(sched, &sched->sched_rq[i]); init_waitqueue_head(&sched->wake_up_worker); init_waitqueue_head(&sched->job_scheduled); @@ -1252,24 +1215,18 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, atomic_set(&sched->_score, 0); atomic64_set(&sched->job_id_count, 0); - /* Each scheduler will run on a seperate kernel thread */ - sched->thread = kthread_run(drm_sched_main, sched, sched->name); - if (IS_ERR(sched->thread)) { - ret = PTR_ERR(sched->thread); - sched->thread = NULL; - DRM_DEV_ERROR(sched->dev, "Failed to create scheduler for %s.\n", name); - goto Out_unroll; - } + + /* Each scheduler will run on a seperate kernel thread */ + sched->thread = kthread_run(drm_sched_main, sched, sched->name); + if (IS_ERR(sched->thread)) { + ret = PTR_ERR(sched->thread); + sched->thread = NULL; + DRM_DEV_ERROR(sched->dev, "Failed to create scheduler for %s.\n", name); + return ret; + } sched->ready = true; return 0; -Out_unroll: - for (--i ; i >= DRM_SCHED_PRIORITY_KERNEL; i--) - kfree(sched->sched_rq[i]); - kfree(sched->sched_rq); - sched->sched_rq = NULL; - drm_err(sched, "%s: Failed to setup GPU scheduler--out of memory\n", __func__); - return ret; } EXPORT_SYMBOL(drm_sched_init); @@ -1288,8 +1245,8 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) if (sched->thread) kthread_stop(sched->thread); - for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { - struct drm_sched_rq *rq = sched->sched_rq[i]; + for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_KERNEL; i--) { + struct drm_sched_rq *rq = &sched->sched_rq[i]; spin_lock(&rq->lock); list_for_each_entry(s_entity, &rq->entities, list) @@ -1300,7 +1257,7 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) */ s_entity->stopped = true; spin_unlock(&rq->lock); - kfree(sched->sched_rq[i]); + } /* Wakeup everyone stuck in drm_sched_entity_flush for this scheduler */ @@ -1310,8 +1267,6 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) cancel_delayed_work_sync(&sched->work_tdr); sched->ready = false; - kfree(sched->sched_rq); - sched->sched_rq = NULL; } EXPORT_SYMBOL(drm_sched_fini); @@ -1337,8 +1292,10 @@ void drm_sched_increase_karma(struct drm_sched_job *bad) */ if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { atomic_inc(&bad->karma); - for (i = DRM_SCHED_PRIORITY_HIGH; i < sched->num_rqs; i++) { - struct drm_sched_rq *rq = sched->sched_rq[i]; + + for (i = DRM_SCHED_PRIORITY_KERNEL; i < DRM_SCHED_PRIORITY_COUNT; + i++) { + struct drm_sched_rq *rq = &sched->sched_rq[i]; spin_lock(&rq->lock); list_for_each_entry_safe(entity, tmp, &rq->entities, list) { diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 39dd0c1d8ee80..f585d23c67c77 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -733,7 +733,6 @@ v3d_sched_init(struct v3d_dev *v3d) ret = drm_sched_init(&v3d->queue[V3D_BIN].sched, &v3d_bin_sched_ops, - DRM_SCHED_PRIORITY_COUNT, hw_jobs_limit, job_hang_limit, msecs_to_jiffies(hang_limit_ms), NULL, NULL, "v3d_bin", v3d->drm.dev); @@ -742,7 +741,6 @@ v3d_sched_init(struct v3d_dev *v3d) ret = drm_sched_init(&v3d->queue[V3D_RENDER].sched, &v3d_render_sched_ops, - DRM_SCHED_PRIORITY_COUNT, hw_jobs_limit, job_hang_limit, msecs_to_jiffies(hang_limit_ms), NULL, NULL, "v3d_render", v3d->drm.dev); @@ -751,7 +749,6 @@ v3d_sched_init(struct v3d_dev *v3d) ret = drm_sched_init(&v3d->queue[V3D_TFU].sched, &v3d_tfu_sched_ops, - DRM_SCHED_PRIORITY_COUNT, hw_jobs_limit, job_hang_limit, msecs_to_jiffies(hang_limit_ms), NULL, NULL, "v3d_tfu", v3d->drm.dev); @@ -761,7 +758,6 @@ v3d_sched_init(struct v3d_dev *v3d) if (v3d_has_csd(v3d)) { ret = drm_sched_init(&v3d->queue[V3D_CSD].sched, &v3d_csd_sched_ops, - DRM_SCHED_PRIORITY_COUNT, hw_jobs_limit, job_hang_limit, msecs_to_jiffies(hang_limit_ms), NULL, NULL, "v3d_csd", v3d->drm.dev); @@ -770,7 +766,6 @@ v3d_sched_init(struct v3d_dev *v3d) ret = drm_sched_init(&v3d->queue[V3D_CACHE_CLEAN].sched, &v3d_cache_clean_sched_ops, - DRM_SCHED_PRIORITY_COUNT, hw_jobs_limit, job_hang_limit, msecs_to_jiffies(hang_limit_ms), NULL, NULL, "v3d_cache_clean", v3d->drm.dev); diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 2dc8bf24aa25e..36ca3137d7b45 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -490,7 +490,6 @@ struct drm_sched_backend_ops { * @credit_count: the current credit count of this scheduler * @timeout: the time after which a job is removed from the scheduler. * @name: name of the ring for which this scheduler is being used. - * @num_rqs: Number of run-queues. This is at most DRM_SCHED_PRIORITY_COUNT, * as there's usually one run-queue per priority, but could be less. * @sched_rq: An allocated array of run-queues of size @num_rqs; * @wake_up_worker: the wait queue on which the scheduler sleeps until a job @@ -521,8 +520,7 @@ struct drm_gpu_scheduler { atomic_t credit_count; long timeout; const char *name; - u32 num_rqs; - struct drm_sched_rq **sched_rq; + struct drm_sched_rq sched_rq[DRM_SCHED_PRIORITY_COUNT]; wait_queue_head_t wake_up_worker; wait_queue_head_t job_scheduled; atomic64_t job_id_count; @@ -541,7 +539,7 @@ struct drm_gpu_scheduler { int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_backend_ops *ops, - u32 num_rqs, u32 credit_limit, unsigned int hang_limit, + u32 credit_limit, unsigned int hang_limit, long timeout, struct workqueue_struct *timeout_wq, atomic_t *score, const char *name, struct device *dev);