Decauple sched threads stop and start and ring mirror list handling from the policy of what to do about the guilty jobs. When stoppping the sched thread and detaching sched fences from non signaled HW fenes wait for all signaled HW fences to complete before rerunning the jobs.
v2: Fix resubmission of guilty job into HW after refactoring.
v4: Full restart for all the jobs, not only from guilty ring. Extract karma increase into standalone function.
v5: Rework waiting for signaled jobs without relying on the job struct itself as those might already be freed for non 'guilty' job's schedulers. Expose karma increase to drivers.
Suggested-by: Christian Koenig Christian.Koenig@amd.com Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- drivers/gpu/drm/scheduler/sched_main.c | 188 +++++++++++++++++++---------- drivers/gpu/drm/v3d/v3d_sched.c | 12 +- include/drm/gpu_scheduler.h | 10 +- 5 files changed, 151 insertions(+), 88 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 8a078f4..a4bd2d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3298,12 +3298,10 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue;
- kthread_park(ring->sched.thread); + drm_sched_stop(&ring->sched, job ? &job->base : NULL);
- if (job && job->base.sched != &ring->sched) - continue; - - drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL); + if(job) + drm_sched_increase_karma(&job->base);
/* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion(ring); @@ -3454,14 +3452,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue;
- /* only need recovery sched of the given job's ring - * or all rings (in the case @job is NULL) - * after above amdgpu_reset accomplished - */ - if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res) - drm_sched_job_recovery(&ring->sched); + if (!adev->asic_reset_res) + drm_sched_resubmit_jobs(&ring->sched);
- kthread_unpark(ring->sched.thread); + drm_sched_start(&ring->sched, !adev->asic_reset_res); }
if (!amdgpu_device_has_dc_support(adev)) { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 49a6763..6f1268f 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) }
/* block scheduler */ - kthread_park(gpu->sched.thread); - drm_sched_hw_job_reset(&gpu->sched, sched_job); + drm_sched_stop(&gpu->sched, sched_job); + + if(sched_job) + drm_sched_increase_karma(sched_job);
/* get the GPU back into the init state */ etnaviv_core_dump(gpu); etnaviv_gpu_recover_hang(gpu);
+ drm_sched_resubmit_jobs(&gpu->sched); + /* restart scheduler after GPU is usable again */ - drm_sched_job_recovery(&gpu->sched); - kthread_unpark(gpu->sched.thread); + drm_sched_start(&gpu->sched, true); }
static void etnaviv_sched_free_job(struct drm_sched_job *sched_job) diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index dbb6906..b5c5bee 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -60,8 +60,6 @@
static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
-static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job); - /** * drm_sched_rq_init - initialize a given run queue struct * @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct work_struct *work) spin_unlock_irqrestore(&sched->job_list_lock, flags); }
+void drm_sched_increase_karma(struct drm_sched_job *bad) +{ + int i; + struct drm_sched_entity *tmp; + struct drm_sched_entity *entity; + struct drm_gpu_scheduler *sched = bad->sched; + + /* don't increase @bad's karma if it's from KERNEL RQ, + * because sometimes GPU hang would cause kernel jobs (like VM updating jobs) + * corrupt but keep in mind that kernel jobs always considered good. + */ + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; + i++) { + struct drm_sched_rq *rq = &sched->sched_rq[i]; + + spin_lock(&rq->lock); + list_for_each_entry_safe(entity, tmp, &rq->entities, list) { + if (bad->s_fence->scheduled.context == + entity->fence_context) { + if (atomic_read(&bad->karma) > + bad->sched->hang_limit) + if (entity->guilty) + atomic_set(entity->guilty, 1); + break; + } + } + spin_unlock(&rq->lock); + if (&entity->list != &rq->entities) + break; + } + } +} +EXPORT_SYMBOL(drm_sched_increase_karma); + /** * drm_sched_hw_job_reset - stop the scheduler if it contains the bad job * @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct work_struct *work) * @bad: bad scheduler job * */ -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) { - struct drm_sched_job *s_job; - struct drm_sched_entity *entity, *tmp; + struct drm_sched_job *s_job, *last_job; unsigned long flags; - int i; + struct dma_fence *wait_fence = NULL; + int r; + + kthread_park(sched->thread);
+ /* + * Verify all the signaled jobs in mirror list are removed from the ring + * by waiting for their respective scheduler fences to signal. + * Continually repeat traversing the ring mirror list until no more signaled + * fences are found + */ +retry_wait: spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { if (s_job->s_fence->parent && @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL; atomic_dec(&sched->hw_rq_count); + } else { + wait_fence = dma_fence_get(&s_job->s_fence->finished); + last_job = s_job; + break; } } - spin_unlock_irqrestore(&sched->job_list_lock, flags);
- if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { - atomic_inc(&bad->karma); - /* don't increase @bad's karma if it's from KERNEL RQ, - * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs) - * corrupt but keep in mind that kernel jobs always considered good. - */ - for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) { - struct drm_sched_rq *rq = &sched->sched_rq[i]; + /* No signaled jobs in the ring, its safe to proceed to ASIC reset */ + if (!wait_fence) { + spin_unlock_irqrestore(&sched->job_list_lock, flags); + goto done; + }
- spin_lock(&rq->lock); - list_for_each_entry_safe(entity, tmp, &rq->entities, list) { - if (bad->s_fence->scheduled.context == entity->fence_context) { - if (atomic_read(&bad->karma) > bad->sched->hang_limit) - if (entity->guilty) - atomic_set(entity->guilty, 1); - break; - } - } - spin_unlock(&rq->lock); - if (&entity->list != &rq->entities) - break; + /* Restore removed cb since removing again already removed cb is undefined */ + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { + if(s_job == last_job) + break; + + if (s_job->s_fence->parent) { + r = dma_fence_add_callback(s_job->s_fence->parent, + &s_job->s_fence->cb, + drm_sched_process_job); + if (r) + DRM_ERROR("fence restore callback failed (%d)\n", + r); } } + spin_unlock_irqrestore(&sched->job_list_lock, flags); + + dma_fence_wait(wait_fence, false); + dma_fence_put(wait_fence); + wait_fence = NULL; + + goto retry_wait; + +done: + return; } -EXPORT_SYMBOL(drm_sched_hw_job_reset); +EXPORT_SYMBOL(drm_sched_stop);
/** * drm_sched_job_recovery - recover jobs after a reset @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset); * @sched: scheduler instance * */ -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) { struct drm_sched_job *s_job, *tmp; - bool found_guilty = false; unsigned long flags; int r;
+ if (!full_recovery) + goto unpark; + spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { struct drm_sched_fence *s_fence = s_job->s_fence; - struct dma_fence *fence; - uint64_t guilty_context; - - if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { - found_guilty = true; - guilty_context = s_job->s_fence->scheduled.context; - } - - if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) - dma_fence_set_error(&s_fence->finished, -ECANCELED); - - spin_unlock_irqrestore(&sched->job_list_lock, flags); - fence = sched->ops->run_job(s_job); - atomic_inc(&sched->hw_rq_count); + struct dma_fence *fence = s_job->s_fence->parent;
if (fence) { - s_fence->parent = dma_fence_get(fence); r = dma_fence_add_callback(fence, &s_fence->cb, drm_sched_process_job); if (r == -ENOENT) @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); - dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(s_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } - spin_lock_irqsave(&sched->job_list_lock, flags); } + drm_sched_start_timeout(sched); spin_unlock_irqrestore(&sched->job_list_lock, flags); + +unpark: + kthread_unpark(sched->thread); +} +EXPORT_SYMBOL(drm_sched_start); + +/** + * drm_sched_resubmit_jobs - helper to relunch job from mirror ring list + * + * @sched: scheduler instance + * + */ +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_job *s_job, *tmp; + uint64_t guilty_context; + bool found_guilty = false; + + /*TODO DO we need spinlock here ? */ + list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { + struct drm_sched_fence *s_fence = s_job->s_fence; + + if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { + found_guilty = true; + guilty_context = s_job->s_fence->scheduled.context; + } + + if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) + dma_fence_set_error(&s_fence->finished, -ECANCELED); + + s_job->s_fence->parent = sched->ops->run_job(s_job); + atomic_inc(&sched->hw_rq_count); + } } -EXPORT_SYMBOL(drm_sched_job_recovery); +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
/** * drm_sched_job_init - init a scheduler job @@ -634,26 +704,14 @@ static int drm_sched_main(void *param) DRM_ERROR("fence add callback failed (%d)\n", r); dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(sched_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - }
wake_up(&sched->job_scheduled); } return 0; }
-static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job) -{ - struct drm_gpu_scheduler *sched = s_job->sched; - - spin_lock(&sched->job_list_lock); - list_del_init(&s_job->node); - spin_unlock(&sched->job_list_lock); -} - /** * drm_sched_init - Init a gpu scheduler instance * diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 445b2ef..f76d9ed 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job) for (q = 0; q < V3D_MAX_QUEUES; q++) { struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
- kthread_park(sched->thread); - drm_sched_hw_job_reset(sched, (sched_job->sched == sched ? + drm_sched_stop(sched, (sched_job->sched == sched ? sched_job : NULL)); + + if(sched_job) + drm_sched_increase_karma(sched_job); }
/* get the GPU back into the init state */ v3d_reset(v3d);
+ for (q = 0; q < V3D_MAX_QUEUES; q++) + drm_sched_resubmit_jobs(sched_job->sched); + /* Unblock schedulers and restart their jobs. */ for (q = 0; q < V3D_MAX_QUEUES; q++) { - drm_sched_job_recovery(&v3d->queue[q].sched); - kthread_unpark(v3d->queue[q].sched.thread); + drm_sched_start(&v3d->queue[q].sched, true); }
mutex_unlock(&v3d->reset_lock); diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 47e1979..5ab2d97 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f); * finished to remove the job from the * @drm_gpu_scheduler.ring_mirror_list. * @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list. + * @finish_node: used in a list to wait on before resetting the scheduler * @id: a unique id assigned to each job scheduled on the scheduler. * @karma: increment on every hang caused by this job. If this exceeds the hang * limit of the scheduler then the job is marked guilty and will not @@ -193,6 +194,7 @@ struct drm_sched_job { struct dma_fence_cb finish_cb; struct work_struct finish_work; struct list_head node; + struct list_head finish_node; uint64_t id; atomic_t karma; enum drm_sched_priority s_priority; @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job, void *owner); void drm_sched_job_cleanup(struct drm_sched_job *job); void drm_sched_wakeup(struct drm_gpu_scheduler *sched); -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, - struct drm_sched_job *job); -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); +void drm_sched_stop(struct drm_gpu_scheduler *sched, + struct drm_sched_job *job); +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); +void drm_sched_increase_karma(struct drm_sched_job *bad); bool drm_sched_dependency_optimized(struct dma_fence* fence, struct drm_sched_entity *entity); void drm_sched_fault(struct drm_gpu_scheduler *sched);
Expedite job deletion from ring mirror list to the HW fence signal callback instead from finish_work, together with waiting for all such fences to signal in drm_sched_stop we garantee that already signaled job will not be processed twice. Remove the sched finish fence callback and just submit finish_work directly from the HW fence callback.
v2: Fix comments. v3: Attach hw fence cb to sched_job v5: Rebase
Suggested-by: Christian Koenig Christian.Koenig@amd.com Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com --- drivers/gpu/drm/scheduler/sched_main.c | 57 +++++++++++++++++----------------- include/drm/gpu_scheduler.h | 6 ++-- 2 files changed, 30 insertions(+), 33 deletions(-)
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index b5c5bee..5f5b187 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -284,8 +284,6 @@ static void drm_sched_job_finish(struct work_struct *work) cancel_delayed_work_sync(&sched->work_tdr);
spin_lock_irqsave(&sched->job_list_lock, flags); - /* remove job from ring_mirror_list */ - list_del_init(&s_job->node); /* queue TDR for next job */ drm_sched_start_timeout(sched); spin_unlock_irqrestore(&sched->job_list_lock, flags); @@ -293,22 +291,11 @@ static void drm_sched_job_finish(struct work_struct *work) sched->ops->free_job(s_job); }
-static void drm_sched_job_finish_cb(struct dma_fence *f, - struct dma_fence_cb *cb) -{ - struct drm_sched_job *job = container_of(cb, struct drm_sched_job, - finish_cb); - schedule_work(&job->finish_work); -} - static void drm_sched_job_begin(struct drm_sched_job *s_job) { struct drm_gpu_scheduler *sched = s_job->sched; unsigned long flags;
- dma_fence_add_callback(&s_job->s_fence->finished, &s_job->finish_cb, - drm_sched_job_finish_cb); - spin_lock_irqsave(&sched->job_list_lock, flags); list_add_tail(&s_job->node, &sched->ring_mirror_list); drm_sched_start_timeout(sched); @@ -396,7 +383,7 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { if (s_job->s_fence->parent && dma_fence_remove_callback(s_job->s_fence->parent, - &s_job->s_fence->cb)) { + &s_job->cb)) { dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL; atomic_dec(&sched->hw_rq_count); @@ -420,7 +407,7 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
if (s_job->s_fence->parent) { r = dma_fence_add_callback(s_job->s_fence->parent, - &s_job->s_fence->cb, + &s_job->cb, drm_sched_process_job); if (r) DRM_ERROR("fence restore callback failed (%d)\n", @@ -449,31 +436,34 @@ EXPORT_SYMBOL(drm_sched_stop); void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) { struct drm_sched_job *s_job, *tmp; - unsigned long flags; int r;
if (!full_recovery) goto unpark;
- spin_lock_irqsave(&sched->job_list_lock, flags); + /* + * Locking the list is not required here as the sched thread is parked + * so no new jobs are being pushed in to HW and in drm_sched_stop we + * flushed all the jobs who were still in mirror list but who already + * signaled and removed them self from the list. Also concurrent + * GPU recovers can't run in parallel. + */ list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { - struct drm_sched_fence *s_fence = s_job->s_fence; struct dma_fence *fence = s_job->s_fence->parent;
if (fence) { - r = dma_fence_add_callback(fence, &s_fence->cb, + r = dma_fence_add_callback(fence, &s_job->cb, drm_sched_process_job); if (r == -ENOENT) - drm_sched_process_job(fence, &s_fence->cb); + drm_sched_process_job(fence, &s_job->cb); else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); } else - drm_sched_process_job(NULL, &s_fence->cb); + drm_sched_process_job(NULL, &s_job->cb); }
drm_sched_start_timeout(sched); - spin_unlock_irqrestore(&sched->job_list_lock, flags);
unpark: kthread_unpark(sched->thread); @@ -622,18 +612,27 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) */ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb) { - struct drm_sched_fence *s_fence = - container_of(cb, struct drm_sched_fence, cb); + struct drm_sched_job *s_job = container_of(cb, struct drm_sched_job, cb); + struct drm_sched_fence *s_fence = s_job->s_fence; struct drm_gpu_scheduler *sched = s_fence->sched; + unsigned long flags; + + cancel_delayed_work(&sched->work_tdr);
- dma_fence_get(&s_fence->finished); atomic_dec(&sched->hw_rq_count); atomic_dec(&sched->num_jobs); + + spin_lock_irqsave(&sched->job_list_lock, flags); + /* remove job from ring_mirror_list */ + list_del_init(&s_job->node); + spin_unlock_irqrestore(&sched->job_list_lock, flags); + drm_sched_fence_finished(s_fence);
trace_drm_sched_process_job(s_fence); - dma_fence_put(&s_fence->finished); wake_up_interruptible(&sched->wake_up_worker); + + schedule_work(&s_job->finish_work); }
/** @@ -696,16 +695,16 @@ static int drm_sched_main(void *param)
if (fence) { s_fence->parent = dma_fence_get(fence); - r = dma_fence_add_callback(fence, &s_fence->cb, + r = dma_fence_add_callback(fence, &sched_job->cb, drm_sched_process_job); if (r == -ENOENT) - drm_sched_process_job(fence, &s_fence->cb); + drm_sched_process_job(fence, &sched_job->cb); else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); dma_fence_put(fence); } else - drm_sched_process_job(NULL, &s_fence->cb); + drm_sched_process_job(NULL, &sched_job->cb);
wake_up(&sched->job_scheduled); } diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 5ab2d97..6621f74 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -138,10 +138,6 @@ struct drm_sched_fence { struct dma_fence finished;
/** - * @cb: the callback for the parent fence below. - */ - struct dma_fence_cb cb; - /** * @parent: the fence returned by &drm_sched_backend_ops.run_job * when scheduling the job on hardware. We signal the * &drm_sched_fence.finished fence once parent is signalled. @@ -182,6 +178,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f); * be scheduled further. * @s_priority: the priority of the job. * @entity: the entity to which this job belongs. + * @cb: the callback for the parent fence in s_fence. * * A job is created by the driver using drm_sched_job_init(), and * should call drm_sched_entity_push_job() once it wants the scheduler @@ -199,6 +196,7 @@ struct drm_sched_job { atomic_t karma; enum drm_sched_priority s_priority; struct drm_sched_entity *entity; + struct dma_fence_cb cb; };
static inline bool drm_sched_invalidate_job(struct drm_sched_job *s_job,
Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
Decauple sched threads stop and start and ring mirror list handling from the policy of what to do about the guilty jobs. When stoppping the sched thread and detaching sched fences from non signaled HW fenes wait for all signaled HW fences to complete before rerunning the jobs.
v2: Fix resubmission of guilty job into HW after refactoring.
v4: Full restart for all the jobs, not only from guilty ring. Extract karma increase into standalone function.
v5: Rework waiting for signaled jobs without relying on the job struct itself as those might already be freed for non 'guilty' job's schedulers. Expose karma increase to drivers.
Suggested-by: Christian Koenig Christian.Koenig@amd.com Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- drivers/gpu/drm/scheduler/sched_main.c | 188 +++++++++++++++++++---------- drivers/gpu/drm/v3d/v3d_sched.c | 12 +- include/drm/gpu_scheduler.h | 10 +- 5 files changed, 151 insertions(+), 88 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 8a078f4..a4bd2d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3298,12 +3298,10 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue;
kthread_park(ring->sched.thread);
drm_sched_stop(&ring->sched, job ? &job->base : NULL);
if (job && job->base.sched != &ring->sched)continue;drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
if(job)drm_sched_increase_karma(&job->base);
Since we dropped the "job && job->base.sched != &ring->sched" check above this will now increase the jobs karma multiple times.
Maybe just move that outside of the loop.
/* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion(ring);@@ -3454,14 +3452,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue;
/* only need recovery sched of the given job's ring* or all rings (in the case @job is NULL)* after above amdgpu_reset accomplished*/if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res)drm_sched_job_recovery(&ring->sched);
if (!adev->asic_reset_res)drm_sched_resubmit_jobs(&ring->sched);
kthread_unpark(ring->sched.thread);
drm_sched_start(&ring->sched, !adev->asic_reset_res);}
if (!amdgpu_device_has_dc_support(adev)) {
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 49a6763..6f1268f 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) }
/* block scheduler */
- kthread_park(gpu->sched.thread);
- drm_sched_hw_job_reset(&gpu->sched, sched_job);
drm_sched_stop(&gpu->sched, sched_job);
if(sched_job)
drm_sched_increase_karma(sched_job);/* get the GPU back into the init state */ etnaviv_core_dump(gpu); etnaviv_gpu_recover_hang(gpu);
drm_sched_resubmit_jobs(&gpu->sched);
/* restart scheduler after GPU is usable again */
- drm_sched_job_recovery(&gpu->sched);
- kthread_unpark(gpu->sched.thread);
drm_sched_start(&gpu->sched, true); }
static void etnaviv_sched_free_job(struct drm_sched_job *sched_job)
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index dbb6906..b5c5bee 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -60,8 +60,6 @@
static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
-static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job);
- /**
- drm_sched_rq_init - initialize a given run queue struct
@@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct work_struct *work) spin_unlock_irqrestore(&sched->job_list_lock, flags); }
Kernel doc here would be nice to have.
+void drm_sched_increase_karma(struct drm_sched_job *bad) +{
- int i;
- struct drm_sched_entity *tmp;
- struct drm_sched_entity *entity;
- struct drm_gpu_scheduler *sched = bad->sched;
- /* don't increase @bad's karma if it's from KERNEL RQ,
* because sometimes GPU hang would cause kernel jobs (like VM updating jobs)* corrupt but keep in mind that kernel jobs always considered good.*/- if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
atomic_inc(&bad->karma);for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;i++) {struct drm_sched_rq *rq = &sched->sched_rq[i];spin_lock(&rq->lock);list_for_each_entry_safe(entity, tmp, &rq->entities, list) {if (bad->s_fence->scheduled.context ==entity->fence_context) {if (atomic_read(&bad->karma) >bad->sched->hang_limit)if (entity->guilty)atomic_set(entity->guilty, 1);break;}}spin_unlock(&rq->lock);if (&entity->list != &rq->entities)break;}- }
+} +EXPORT_SYMBOL(drm_sched_increase_karma);
- /**
- drm_sched_hw_job_reset - stop the scheduler if it contains the bad job
@@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct work_struct *work)
- @bad: bad scheduler job
*/ -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) {
- struct drm_sched_job *s_job;
- struct drm_sched_entity *entity, *tmp;
- struct drm_sched_job *s_job, *last_job; unsigned long flags;
- int i;
struct dma_fence *wait_fence = NULL;
int r;
kthread_park(sched->thread);
/*
* Verify all the signaled jobs in mirror list are removed from the ring* by waiting for their respective scheduler fences to signal.* Continually repeat traversing the ring mirror list until no more signaled* fences are found*/+retry_wait: spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { if (s_job->s_fence->parent && @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL; atomic_dec(&sched->hw_rq_count);
} else {wait_fence = dma_fence_get(&s_job->s_fence->finished);last_job = s_job;break;
spin_unlock_irqrestore(&sched->job_list_lock, flags);
if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
atomic_inc(&bad->karma);/* don't increase @bad's karma if it's from KERNEL RQ,* becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs)* corrupt but keep in mind that kernel jobs always considered good.*/for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) {struct drm_sched_rq *rq = &sched->sched_rq[i];
- /* No signaled jobs in the ring, its safe to proceed to ASIC reset */
- if (!wait_fence) {
spin_unlock_irqrestore(&sched->job_list_lock, flags);goto done;- }
spin_lock(&rq->lock);list_for_each_entry_safe(entity, tmp, &rq->entities, list) {if (bad->s_fence->scheduled.context == entity->fence_context) {if (atomic_read(&bad->karma) > bad->sched->hang_limit)if (entity->guilty)atomic_set(entity->guilty, 1);break;}}spin_unlock(&rq->lock);if (&entity->list != &rq->entities)break;
- /* Restore removed cb since removing again already removed cb is undefined */
- list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
if(s_job == last_job)break;
Need to double check after the holidays, but you should be able to use list_for_each_entry_continue here.
if (s_job->s_fence->parent) {r = dma_fence_add_callback(s_job->s_fence->parent,&s_job->s_fence->cb,drm_sched_process_job);if (r)DRM_ERROR("fence restore callback failed (%d)\n",r);
When you fail to add the callback this means that you need to call call drm_sched_process_job manually here.
}}
- spin_unlock_irqrestore(&sched->job_list_lock, flags);
- dma_fence_wait(wait_fence, false);
- dma_fence_put(wait_fence);
- wait_fence = NULL;
- goto retry_wait;
+done:
- return;
Drop the done: label and return directly above.
Apart from all those nit picks that starts to look like it should work, Christian.
} -EXPORT_SYMBOL(drm_sched_hw_job_reset); +EXPORT_SYMBOL(drm_sched_stop);
/**
- drm_sched_job_recovery - recover jobs after a reset
@@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset);
- @sched: scheduler instance
*/ -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) { struct drm_sched_job *s_job, *tmp;
- bool found_guilty = false; unsigned long flags; int r;
- if (!full_recovery)
goto unpark;- spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { struct drm_sched_fence *s_fence = s_job->s_fence;
struct dma_fence *fence;uint64_t guilty_context;if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {found_guilty = true;guilty_context = s_job->s_fence->scheduled.context;}if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)dma_fence_set_error(&s_fence->finished, -ECANCELED);spin_unlock_irqrestore(&sched->job_list_lock, flags);fence = sched->ops->run_job(s_job);atomic_inc(&sched->hw_rq_count);
struct dma_fence *fence = s_job->s_fence->parent;if (fence) {
s_fence->parent = dma_fence_get(fence); r = dma_fence_add_callback(fence, &s_fence->cb, drm_sched_process_job); if (r == -ENOENT)@@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) else if (r) DRM_ERROR("fence add callback failed (%d)\n", r);
dma_fence_put(fence);} else {if (s_fence->finished.error < 0)drm_sched_expel_job_unlocked(s_job);
} else drm_sched_process_job(NULL, &s_fence->cb);
}spin_lock_irqsave(&sched->job_list_lock, flags);
- drm_sched_start_timeout(sched); spin_unlock_irqrestore(&sched->job_list_lock, flags);
+unpark:
- kthread_unpark(sched->thread);
+} +EXPORT_SYMBOL(drm_sched_start);
+/**
- drm_sched_resubmit_jobs - helper to relunch job from mirror ring list
- @sched: scheduler instance
- */
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) +{
- struct drm_sched_job *s_job, *tmp;
- uint64_t guilty_context;
- bool found_guilty = false;
- /*TODO DO we need spinlock here ? */
- list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
struct drm_sched_fence *s_fence = s_job->s_fence;if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {found_guilty = true;guilty_context = s_job->s_fence->scheduled.context;}if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)dma_fence_set_error(&s_fence->finished, -ECANCELED);s_job->s_fence->parent = sched->ops->run_job(s_job);atomic_inc(&sched->hw_rq_count);- } }
-EXPORT_SYMBOL(drm_sched_job_recovery); +EXPORT_SYMBOL(drm_sched_resubmit_jobs);
/**
- drm_sched_job_init - init a scheduler job
@@ -634,26 +704,14 @@ static int drm_sched_main(void *param) DRM_ERROR("fence add callback failed (%d)\n", r); dma_fence_put(fence);
} else {if (s_fence->finished.error < 0)drm_sched_expel_job_unlocked(sched_job);
} else drm_sched_process_job(NULL, &s_fence->cb);
}wake_up(&sched->job_scheduled); } return 0; }
-static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job) -{
- struct drm_gpu_scheduler *sched = s_job->sched;
- spin_lock(&sched->job_list_lock);
- list_del_init(&s_job->node);
- spin_unlock(&sched->job_list_lock);
-}
- /**
- drm_sched_init - Init a gpu scheduler instance
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 445b2ef..f76d9ed 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job) for (q = 0; q < V3D_MAX_QUEUES; q++) { struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
kthread_park(sched->thread);drm_sched_hw_job_reset(sched, (sched_job->sched == sched ?
drm_sched_stop(sched, (sched_job->sched == sched ? sched_job : NULL));if(sched_job)drm_sched_increase_karma(sched_job);}
/* get the GPU back into the init state */ v3d_reset(v3d);
for (q = 0; q < V3D_MAX_QUEUES; q++)
drm_sched_resubmit_jobs(sched_job->sched);/* Unblock schedulers and restart their jobs. */ for (q = 0; q < V3D_MAX_QUEUES; q++) {
drm_sched_job_recovery(&v3d->queue[q].sched);kthread_unpark(v3d->queue[q].sched.thread);
drm_sched_start(&v3d->queue[q].sched, true);}
mutex_unlock(&v3d->reset_lock);
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 47e1979..5ab2d97 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f);
finished to remove the job from the@drm_gpu_scheduler.ring_mirror_list.- @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list.
- @finish_node: used in a list to wait on before resetting the scheduler
- @id: a unique id assigned to each job scheduled on the scheduler.
- @karma: increment on every hang caused by this job. If this exceeds the hang
limit of the scheduler then the job is marked guilty and will not@@ -193,6 +194,7 @@ struct drm_sched_job { struct dma_fence_cb finish_cb; struct work_struct finish_work; struct list_head node;
- struct list_head finish_node; uint64_t id; atomic_t karma; enum drm_sched_priority s_priority;
@@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job, void *owner); void drm_sched_job_cleanup(struct drm_sched_job *job); void drm_sched_wakeup(struct drm_gpu_scheduler *sched); -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
struct drm_sched_job *job);-void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); +void drm_sched_stop(struct drm_gpu_scheduler *sched,
struct drm_sched_job *job);+void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); +void drm_sched_increase_karma(struct drm_sched_job *bad); bool drm_sched_dependency_optimized(struct dma_fence* fence, struct drm_sched_entity *entity); void drm_sched_fault(struct drm_gpu_scheduler *sched);
On 12/21/2018 01:37 PM, Christian König wrote:
Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
Decauple sched threads stop and start and ring mirror list handling from the policy of what to do about the guilty jobs. When stoppping the sched thread and detaching sched fences from non signaled HW fenes wait for all signaled HW fences to complete before rerunning the jobs.
v2: Fix resubmission of guilty job into HW after refactoring.
v4: Full restart for all the jobs, not only from guilty ring. Extract karma increase into standalone function.
v5: Rework waiting for signaled jobs without relying on the job struct itself as those might already be freed for non 'guilty' job's schedulers. Expose karma increase to drivers.
Suggested-by: Christian Koenig Christian.Koenig@amd.com Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- drivers/gpu/drm/scheduler/sched_main.c | 188 +++++++++++++++++++---------- drivers/gpu/drm/v3d/v3d_sched.c | 12 +- include/drm/gpu_scheduler.h | 10 +- 5 files changed, 151 insertions(+), 88 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 8a078f4..a4bd2d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3298,12 +3298,10 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - kthread_park(ring->sched.thread); + drm_sched_stop(&ring->sched, job ? &job->base : NULL); - if (job && job->base.sched != &ring->sched) - continue;
- drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL); + if(job) + drm_sched_increase_karma(&job->base);
Since we dropped the "job && job->base.sched != &ring->sched" check above this will now increase the jobs karma multiple times.
Maybe just move that outside of the loop.
/* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion(ring); @@ -3454,14 +3452,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - /* only need recovery sched of the given job's ring - * or all rings (in the case @job is NULL) - * after above amdgpu_reset accomplished - */ - if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res) - drm_sched_job_recovery(&ring->sched); + if (!adev->asic_reset_res) + drm_sched_resubmit_jobs(&ring->sched); - kthread_unpark(ring->sched.thread); + drm_sched_start(&ring->sched, !adev->asic_reset_res); } if (!amdgpu_device_has_dc_support(adev)) { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 49a6763..6f1268f 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) } /* block scheduler */ - kthread_park(gpu->sched.thread); - drm_sched_hw_job_reset(&gpu->sched, sched_job); + drm_sched_stop(&gpu->sched, sched_job);
+ if(sched_job) + drm_sched_increase_karma(sched_job); /* get the GPU back into the init state */ etnaviv_core_dump(gpu); etnaviv_gpu_recover_hang(gpu); + drm_sched_resubmit_jobs(&gpu->sched);
/* restart scheduler after GPU is usable again */ - drm_sched_job_recovery(&gpu->sched); - kthread_unpark(gpu->sched.thread); + drm_sched_start(&gpu->sched, true); } static void etnaviv_sched_free_job(struct drm_sched_job *sched_job) diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index dbb6906..b5c5bee 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -60,8 +60,6 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb); -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job);
/** * drm_sched_rq_init - initialize a given run queue struct * @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct work_struct *work) spin_unlock_irqrestore(&sched->job_list_lock, flags); }
Kernel doc here would be nice to have.
+void drm_sched_increase_karma(struct drm_sched_job *bad) +{ + int i; + struct drm_sched_entity *tmp; + struct drm_sched_entity *entity; + struct drm_gpu_scheduler *sched = bad->sched;
+ /* don't increase @bad's karma if it's from KERNEL RQ, + * because sometimes GPU hang would cause kernel jobs (like VM updating jobs) + * corrupt but keep in mind that kernel jobs always considered good. + */ + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; + i++) { + struct drm_sched_rq *rq = &sched->sched_rq[i];
+ spin_lock(&rq->lock); + list_for_each_entry_safe(entity, tmp, &rq->entities, list) { + if (bad->s_fence->scheduled.context == + entity->fence_context) { + if (atomic_read(&bad->karma) > + bad->sched->hang_limit) + if (entity->guilty) + atomic_set(entity->guilty, 1); + break; + } + } + spin_unlock(&rq->lock); + if (&entity->list != &rq->entities) + break; + } + } +} +EXPORT_SYMBOL(drm_sched_increase_karma);
/** * drm_sched_hw_job_reset - stop the scheduler if it contains the bad job * @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct work_struct *work) * @bad: bad scheduler job * */ -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) { - struct drm_sched_job *s_job; - struct drm_sched_entity *entity, *tmp; + struct drm_sched_job *s_job, *last_job; unsigned long flags; - int i; + struct dma_fence *wait_fence = NULL; + int r;
+ kthread_park(sched->thread); + /* + * Verify all the signaled jobs in mirror list are removed from the ring + * by waiting for their respective scheduler fences to signal. + * Continually repeat traversing the ring mirror list until no more signaled + * fences are found + */ +retry_wait: spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { if (s_job->s_fence->parent && @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL; atomic_dec(&sched->hw_rq_count); + } else { + wait_fence = dma_fence_get(&s_job->s_fence->finished); + last_job = s_job; + break; } } - spin_unlock_irqrestore(&sched->job_list_lock, flags); - if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { - atomic_inc(&bad->karma); - /* don't increase @bad's karma if it's from KERNEL RQ, - * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs) - * corrupt but keep in mind that kernel jobs always considered good. - */ - for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) { - struct drm_sched_rq *rq = &sched->sched_rq[i]; + /* No signaled jobs in the ring, its safe to proceed to ASIC reset */ + if (!wait_fence) { + spin_unlock_irqrestore(&sched->job_list_lock, flags); + goto done; + } - spin_lock(&rq->lock); - list_for_each_entry_safe(entity, tmp, &rq->entities, list) { - if (bad->s_fence->scheduled.context == entity->fence_context) { - if (atomic_read(&bad->karma) > bad->sched->hang_limit) - if (entity->guilty) - atomic_set(entity->guilty, 1); - break; - } - } - spin_unlock(&rq->lock); - if (&entity->list != &rq->entities) - break; + /* Restore removed cb since removing again already removed cb is undefined */ + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { + if(s_job == last_job) + break;
Need to double check after the holidays, but you should be able to use list_for_each_entry_continue here.
I think it should work - kind of traversing back on all the jobs we just removed their callbacks.
Andrey
+ if (s_job->s_fence->parent) { + r = dma_fence_add_callback(s_job->s_fence->parent, + &s_job->s_fence->cb, + drm_sched_process_job); + if (r) + DRM_ERROR("fence restore callback failed (%d)\n", + r);
When you fail to add the callback this means that you need to call call drm_sched_process_job manually here.
} } + spin_unlock_irqrestore(&sched->job_list_lock, flags);
+ dma_fence_wait(wait_fence, false); + dma_fence_put(wait_fence); + wait_fence = NULL;
+ goto retry_wait;
+done: + return;
Drop the done: label and return directly above.
Apart from all those nit picks that starts to look like it should work, Christian.
} -EXPORT_SYMBOL(drm_sched_hw_job_reset); +EXPORT_SYMBOL(drm_sched_stop); /** * drm_sched_job_recovery - recover jobs after a reset @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset); * @sched: scheduler instance * */ -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) { struct drm_sched_job *s_job, *tmp; - bool found_guilty = false; unsigned long flags; int r; + if (!full_recovery) + goto unpark;
spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { struct drm_sched_fence *s_fence = s_job->s_fence; - struct dma_fence *fence; - uint64_t guilty_context;
- if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { - found_guilty = true; - guilty_context = s_job->s_fence->scheduled.context; - }
- if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) - dma_fence_set_error(&s_fence->finished, -ECANCELED);
- spin_unlock_irqrestore(&sched->job_list_lock, flags); - fence = sched->ops->run_job(s_job); - atomic_inc(&sched->hw_rq_count); + struct dma_fence *fence = s_job->s_fence->parent; if (fence) { - s_fence->parent = dma_fence_get(fence); r = dma_fence_add_callback(fence, &s_fence->cb, drm_sched_process_job); if (r == -ENOENT) @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); - dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(s_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } - spin_lock_irqsave(&sched->job_list_lock, flags); }
drm_sched_start_timeout(sched); spin_unlock_irqrestore(&sched->job_list_lock, flags);
+unpark: + kthread_unpark(sched->thread); +} +EXPORT_SYMBOL(drm_sched_start);
+/**
- drm_sched_resubmit_jobs - helper to relunch job from mirror ring
list
- @sched: scheduler instance
- */
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_job *s_job, *tmp; + uint64_t guilty_context; + bool found_guilty = false;
+ /*TODO DO we need spinlock here ? */ + list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { + struct drm_sched_fence *s_fence = s_job->s_fence;
+ if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { + found_guilty = true; + guilty_context = s_job->s_fence->scheduled.context; + }
+ if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) + dma_fence_set_error(&s_fence->finished, -ECANCELED);
+ s_job->s_fence->parent = sched->ops->run_job(s_job); + atomic_inc(&sched->hw_rq_count); + } } -EXPORT_SYMBOL(drm_sched_job_recovery); +EXPORT_SYMBOL(drm_sched_resubmit_jobs); /** * drm_sched_job_init - init a scheduler job @@ -634,26 +704,14 @@ static int drm_sched_main(void *param) DRM_ERROR("fence add callback failed (%d)\n", r); dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(sched_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } wake_up(&sched->job_scheduled); } return 0; } -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job) -{ - struct drm_gpu_scheduler *sched = s_job->sched;
- spin_lock(&sched->job_list_lock); - list_del_init(&s_job->node); - spin_unlock(&sched->job_list_lock); -}
/** * drm_sched_init - Init a gpu scheduler instance * diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 445b2ef..f76d9ed 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job) for (q = 0; q < V3D_MAX_QUEUES; q++) { struct drm_gpu_scheduler *sched = &v3d->queue[q].sched; - kthread_park(sched->thread); - drm_sched_hw_job_reset(sched, (sched_job->sched == sched ? + drm_sched_stop(sched, (sched_job->sched == sched ? sched_job : NULL));
+ if(sched_job) + drm_sched_increase_karma(sched_job); } /* get the GPU back into the init state */ v3d_reset(v3d); + for (q = 0; q < V3D_MAX_QUEUES; q++) + drm_sched_resubmit_jobs(sched_job->sched);
/* Unblock schedulers and restart their jobs. */ for (q = 0; q < V3D_MAX_QUEUES; q++) { - drm_sched_job_recovery(&v3d->queue[q].sched); - kthread_unpark(v3d->queue[q].sched.thread); + drm_sched_start(&v3d->queue[q].sched, true); } mutex_unlock(&v3d->reset_lock); diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 47e1979..5ab2d97 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f); * finished to remove the job from the * @drm_gpu_scheduler.ring_mirror_list. * @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list.
- @finish_node: used in a list to wait on before resetting the
scheduler * @id: a unique id assigned to each job scheduled on the scheduler. * @karma: increment on every hang caused by this job. If this exceeds the hang * limit of the scheduler then the job is marked guilty and will not @@ -193,6 +194,7 @@ struct drm_sched_job { struct dma_fence_cb finish_cb; struct work_struct finish_work; struct list_head node; + struct list_head finish_node; uint64_t id; atomic_t karma; enum drm_sched_priority s_priority; @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job, void *owner); void drm_sched_job_cleanup(struct drm_sched_job *job); void drm_sched_wakeup(struct drm_gpu_scheduler *sched); -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, - struct drm_sched_job *job); -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); +void drm_sched_stop(struct drm_gpu_scheduler *sched, + struct drm_sched_job *job); +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); +void drm_sched_increase_karma(struct drm_sched_job *bad); bool drm_sched_dependency_optimized(struct dma_fence* fence, struct drm_sched_entity *entity); void drm_sched_fault(struct drm_gpu_scheduler *sched);
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
On 12/21/2018 01:37 PM, Christian König wrote:
Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
Decauple sched threads stop and start and ring mirror list handling from the policy of what to do about the guilty jobs. When stoppping the sched thread and detaching sched fences from non signaled HW fenes wait for all signaled HW fences to complete before rerunning the jobs.
v2: Fix resubmission of guilty job into HW after refactoring.
v4: Full restart for all the jobs, not only from guilty ring. Extract karma increase into standalone function.
v5: Rework waiting for signaled jobs without relying on the job struct itself as those might already be freed for non 'guilty' job's schedulers. Expose karma increase to drivers.
Suggested-by: Christian Koenig Christian.Koenig@amd.com Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- drivers/gpu/drm/scheduler/sched_main.c | 188 +++++++++++++++++++---------- drivers/gpu/drm/v3d/v3d_sched.c | 12 +- include/drm/gpu_scheduler.h | 10 +- 5 files changed, 151 insertions(+), 88 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 8a078f4..a4bd2d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3298,12 +3298,10 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - kthread_park(ring->sched.thread); + drm_sched_stop(&ring->sched, job ? &job->base : NULL); - if (job && job->base.sched != &ring->sched) - continue;
- drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL); + if(job) + drm_sched_increase_karma(&job->base);
Since we dropped the "job && job->base.sched != &ring->sched" check above this will now increase the jobs karma multiple times.
Maybe just move that outside of the loop.
/* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion(ring); @@ -3454,14 +3452,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - /* only need recovery sched of the given job's ring - * or all rings (in the case @job is NULL) - * after above amdgpu_reset accomplished - */ - if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res) - drm_sched_job_recovery(&ring->sched); + if (!adev->asic_reset_res) + drm_sched_resubmit_jobs(&ring->sched); - kthread_unpark(ring->sched.thread); + drm_sched_start(&ring->sched, !adev->asic_reset_res); } if (!amdgpu_device_has_dc_support(adev)) { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 49a6763..6f1268f 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) } /* block scheduler */ - kthread_park(gpu->sched.thread); - drm_sched_hw_job_reset(&gpu->sched, sched_job); + drm_sched_stop(&gpu->sched, sched_job);
+ if(sched_job) + drm_sched_increase_karma(sched_job); /* get the GPU back into the init state */ etnaviv_core_dump(gpu); etnaviv_gpu_recover_hang(gpu); + drm_sched_resubmit_jobs(&gpu->sched);
/* restart scheduler after GPU is usable again */ - drm_sched_job_recovery(&gpu->sched); - kthread_unpark(gpu->sched.thread); + drm_sched_start(&gpu->sched, true); } static void etnaviv_sched_free_job(struct drm_sched_job *sched_job) diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index dbb6906..b5c5bee 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -60,8 +60,6 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb); -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job);
/** * drm_sched_rq_init - initialize a given run queue struct * @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct work_struct *work) spin_unlock_irqrestore(&sched->job_list_lock, flags); }
Kernel doc here would be nice to have.
+void drm_sched_increase_karma(struct drm_sched_job *bad) +{ + int i; + struct drm_sched_entity *tmp; + struct drm_sched_entity *entity; + struct drm_gpu_scheduler *sched = bad->sched;
+ /* don't increase @bad's karma if it's from KERNEL RQ, + * because sometimes GPU hang would cause kernel jobs (like VM updating jobs) + * corrupt but keep in mind that kernel jobs always considered good. + */ + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; + i++) { + struct drm_sched_rq *rq = &sched->sched_rq[i];
+ spin_lock(&rq->lock); + list_for_each_entry_safe(entity, tmp, &rq->entities, list) { + if (bad->s_fence->scheduled.context == + entity->fence_context) { + if (atomic_read(&bad->karma) > + bad->sched->hang_limit) + if (entity->guilty) + atomic_set(entity->guilty, 1); + break; + } + } + spin_unlock(&rq->lock); + if (&entity->list != &rq->entities) + break; + } + } +} +EXPORT_SYMBOL(drm_sched_increase_karma);
/** * drm_sched_hw_job_reset - stop the scheduler if it contains the bad job * @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct work_struct *work) * @bad: bad scheduler job * */ -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) { - struct drm_sched_job *s_job; - struct drm_sched_entity *entity, *tmp; + struct drm_sched_job *s_job, *last_job; unsigned long flags; - int i; + struct dma_fence *wait_fence = NULL; + int r;
+ kthread_park(sched->thread); + /* + * Verify all the signaled jobs in mirror list are removed from the ring + * by waiting for their respective scheduler fences to signal. + * Continually repeat traversing the ring mirror list until no more signaled + * fences are found + */ +retry_wait: spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { if (s_job->s_fence->parent && @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL; atomic_dec(&sched->hw_rq_count); + } else { + wait_fence = dma_fence_get(&s_job->s_fence->finished); + last_job = s_job; + break; } } - spin_unlock_irqrestore(&sched->job_list_lock, flags); - if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { - atomic_inc(&bad->karma); - /* don't increase @bad's karma if it's from KERNEL RQ, - * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs) - * corrupt but keep in mind that kernel jobs always considered good. - */ - for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) { - struct drm_sched_rq *rq = &sched->sched_rq[i]; + /* No signaled jobs in the ring, its safe to proceed to ASIC reset */ + if (!wait_fence) { + spin_unlock_irqrestore(&sched->job_list_lock, flags); + goto done; + } - spin_lock(&rq->lock); - list_for_each_entry_safe(entity, tmp, &rq->entities, list) { - if (bad->s_fence->scheduled.context == entity->fence_context) { - if (atomic_read(&bad->karma) > bad->sched->hang_limit) - if (entity->guilty) - atomic_set(entity->guilty, 1); - break; - } - } - spin_unlock(&rq->lock); - if (&entity->list != &rq->entities) - break; + /* Restore removed cb since removing again already removed cb is undefined */ + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { + if(s_job == last_job) + break;
Need to double check after the holidays, but you should be able to use list_for_each_entry_continue here.
I think it should work - kind of traversing back on all the jobs we just removed their callbacks.
Wrapping your head around stuff again after the holidays sometimes shows new problems we have completely missed :)
Adding the callbacks again won't work because we have freed up our reference to the hardware fence above:
dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL;
We need to drop this or we would never be able to re-add the fences.
But I'm still not sure if we shouldn't rely on the correct order of signaling and simplify this instead.
Regards, Christian.
Andrey
+ if (s_job->s_fence->parent) { + r = dma_fence_add_callback(s_job->s_fence->parent, + &s_job->s_fence->cb, + drm_sched_process_job); + if (r) + DRM_ERROR("fence restore callback failed (%d)\n", + r);
When you fail to add the callback this means that you need to call call drm_sched_process_job manually here.
} } + spin_unlock_irqrestore(&sched->job_list_lock, flags);
+ dma_fence_wait(wait_fence, false); + dma_fence_put(wait_fence); + wait_fence = NULL;
+ goto retry_wait;
+done: + return;
Drop the done: label and return directly above.
Apart from all those nit picks that starts to look like it should work, Christian.
} -EXPORT_SYMBOL(drm_sched_hw_job_reset); +EXPORT_SYMBOL(drm_sched_stop); /** * drm_sched_job_recovery - recover jobs after a reset @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset); * @sched: scheduler instance * */ -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) { struct drm_sched_job *s_job, *tmp; - bool found_guilty = false; unsigned long flags; int r; + if (!full_recovery) + goto unpark;
spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { struct drm_sched_fence *s_fence = s_job->s_fence; - struct dma_fence *fence; - uint64_t guilty_context;
- if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { - found_guilty = true; - guilty_context = s_job->s_fence->scheduled.context; - }
- if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) - dma_fence_set_error(&s_fence->finished, -ECANCELED);
- spin_unlock_irqrestore(&sched->job_list_lock, flags); - fence = sched->ops->run_job(s_job); - atomic_inc(&sched->hw_rq_count); + struct dma_fence *fence = s_job->s_fence->parent; if (fence) { - s_fence->parent = dma_fence_get(fence); r = dma_fence_add_callback(fence, &s_fence->cb, drm_sched_process_job); if (r == -ENOENT) @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); - dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(s_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } - spin_lock_irqsave(&sched->job_list_lock, flags); }
drm_sched_start_timeout(sched); spin_unlock_irqrestore(&sched->job_list_lock, flags);
+unpark: + kthread_unpark(sched->thread); +} +EXPORT_SYMBOL(drm_sched_start);
+/**
- drm_sched_resubmit_jobs - helper to relunch job from mirror ring
list
- @sched: scheduler instance
- */
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_job *s_job, *tmp; + uint64_t guilty_context; + bool found_guilty = false;
+ /*TODO DO we need spinlock here ? */ + list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { + struct drm_sched_fence *s_fence = s_job->s_fence;
+ if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { + found_guilty = true; + guilty_context = s_job->s_fence->scheduled.context; + }
+ if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) + dma_fence_set_error(&s_fence->finished, -ECANCELED);
+ s_job->s_fence->parent = sched->ops->run_job(s_job); + atomic_inc(&sched->hw_rq_count); + } } -EXPORT_SYMBOL(drm_sched_job_recovery); +EXPORT_SYMBOL(drm_sched_resubmit_jobs); /** * drm_sched_job_init - init a scheduler job @@ -634,26 +704,14 @@ static int drm_sched_main(void *param) DRM_ERROR("fence add callback failed (%d)\n", r); dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(sched_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } wake_up(&sched->job_scheduled); } return 0; } -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job) -{ - struct drm_gpu_scheduler *sched = s_job->sched;
- spin_lock(&sched->job_list_lock); - list_del_init(&s_job->node); - spin_unlock(&sched->job_list_lock); -}
/** * drm_sched_init - Init a gpu scheduler instance * diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 445b2ef..f76d9ed 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job) for (q = 0; q < V3D_MAX_QUEUES; q++) { struct drm_gpu_scheduler *sched = &v3d->queue[q].sched; - kthread_park(sched->thread); - drm_sched_hw_job_reset(sched, (sched_job->sched == sched ? + drm_sched_stop(sched, (sched_job->sched == sched ? sched_job : NULL));
+ if(sched_job) + drm_sched_increase_karma(sched_job); } /* get the GPU back into the init state */ v3d_reset(v3d); + for (q = 0; q < V3D_MAX_QUEUES; q++) + drm_sched_resubmit_jobs(sched_job->sched);
/* Unblock schedulers and restart their jobs. */ for (q = 0; q < V3D_MAX_QUEUES; q++) { - drm_sched_job_recovery(&v3d->queue[q].sched); - kthread_unpark(v3d->queue[q].sched.thread); + drm_sched_start(&v3d->queue[q].sched, true); } mutex_unlock(&v3d->reset_lock); diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 47e1979..5ab2d97 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f); * finished to remove the job from the * @drm_gpu_scheduler.ring_mirror_list. * @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list.
- @finish_node: used in a list to wait on before resetting the
scheduler * @id: a unique id assigned to each job scheduled on the scheduler. * @karma: increment on every hang caused by this job. If this exceeds the hang * limit of the scheduler then the job is marked guilty and will not @@ -193,6 +194,7 @@ struct drm_sched_job { struct dma_fence_cb finish_cb; struct work_struct finish_work; struct list_head node; + struct list_head finish_node; uint64_t id; atomic_t karma; enum drm_sched_priority s_priority; @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job, void *owner); void drm_sched_job_cleanup(struct drm_sched_job *job); void drm_sched_wakeup(struct drm_gpu_scheduler *sched); -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, - struct drm_sched_job *job); -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); +void drm_sched_stop(struct drm_gpu_scheduler *sched, + struct drm_sched_job *job); +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); +void drm_sched_increase_karma(struct drm_sched_job *bad); bool drm_sched_dependency_optimized(struct dma_fence* fence, struct drm_sched_entity *entity); void drm_sched_fault(struct drm_gpu_scheduler *sched);
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/03/2019 03:54 AM, Koenig, Christian wrote:
Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
On 12/21/2018 01:37 PM, Christian König wrote:
Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
Decauple sched threads stop and start and ring mirror list handling from the policy of what to do about the guilty jobs. When stoppping the sched thread and detaching sched fences from non signaled HW fenes wait for all signaled HW fences to complete before rerunning the jobs.
v2: Fix resubmission of guilty job into HW after refactoring.
v4: Full restart for all the jobs, not only from guilty ring. Extract karma increase into standalone function.
v5: Rework waiting for signaled jobs without relying on the job struct itself as those might already be freed for non 'guilty' job's schedulers. Expose karma increase to drivers.
Suggested-by: Christian Koenig Christian.Koenig@amd.com Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- drivers/gpu/drm/scheduler/sched_main.c | 188 +++++++++++++++++++---------- drivers/gpu/drm/v3d/v3d_sched.c | 12 +- include/drm/gpu_scheduler.h | 10 +- 5 files changed, 151 insertions(+), 88 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 8a078f4..a4bd2d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3298,12 +3298,10 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - kthread_park(ring->sched.thread); + drm_sched_stop(&ring->sched, job ? &job->base : NULL); - if (job && job->base.sched != &ring->sched) - continue;
- drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL); + if(job) + drm_sched_increase_karma(&job->base);
Since we dropped the "job && job->base.sched != &ring->sched" check above this will now increase the jobs karma multiple times.
Maybe just move that outside of the loop.
/* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion(ring); @@ -3454,14 +3452,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - /* only need recovery sched of the given job's ring - * or all rings (in the case @job is NULL) - * after above amdgpu_reset accomplished - */ - if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res) - drm_sched_job_recovery(&ring->sched); + if (!adev->asic_reset_res) + drm_sched_resubmit_jobs(&ring->sched); - kthread_unpark(ring->sched.thread); + drm_sched_start(&ring->sched, !adev->asic_reset_res); } if (!amdgpu_device_has_dc_support(adev)) { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 49a6763..6f1268f 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) } /* block scheduler */ - kthread_park(gpu->sched.thread); - drm_sched_hw_job_reset(&gpu->sched, sched_job); + drm_sched_stop(&gpu->sched, sched_job);
+ if(sched_job) + drm_sched_increase_karma(sched_job); /* get the GPU back into the init state */ etnaviv_core_dump(gpu); etnaviv_gpu_recover_hang(gpu); + drm_sched_resubmit_jobs(&gpu->sched);
- /* restart scheduler after GPU is usable again */
- drm_sched_job_recovery(&gpu->sched); - kthread_unpark(gpu->sched.thread); + drm_sched_start(&gpu->sched, true); } static void etnaviv_sched_free_job(struct drm_sched_job *sched_job) diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index dbb6906..b5c5bee 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -60,8 +60,6 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb); -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job);
- /** * drm_sched_rq_init - initialize a given run queue struct *
@@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct work_struct *work) spin_unlock_irqrestore(&sched->job_list_lock, flags); }
Kernel doc here would be nice to have.
+void drm_sched_increase_karma(struct drm_sched_job *bad) +{ + int i; + struct drm_sched_entity *tmp; + struct drm_sched_entity *entity; + struct drm_gpu_scheduler *sched = bad->sched;
+ /* don't increase @bad's karma if it's from KERNEL RQ, + * because sometimes GPU hang would cause kernel jobs (like VM updating jobs) + * corrupt but keep in mind that kernel jobs always considered good. + */ + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; + i++) { + struct drm_sched_rq *rq = &sched->sched_rq[i];
+ spin_lock(&rq->lock); + list_for_each_entry_safe(entity, tmp, &rq->entities, list) { + if (bad->s_fence->scheduled.context == + entity->fence_context) { + if (atomic_read(&bad->karma) > + bad->sched->hang_limit) + if (entity->guilty) + atomic_set(entity->guilty, 1); + break; + } + } + spin_unlock(&rq->lock); + if (&entity->list != &rq->entities) + break; + } + } +} +EXPORT_SYMBOL(drm_sched_increase_karma);
- /** * drm_sched_hw_job_reset - stop the scheduler if it contains the
bad job * @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct work_struct *work) * @bad: bad scheduler job * */ -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) { - struct drm_sched_job *s_job; - struct drm_sched_entity *entity, *tmp; + struct drm_sched_job *s_job, *last_job; unsigned long flags; - int i; + struct dma_fence *wait_fence = NULL; + int r;
+ kthread_park(sched->thread); + /* + * Verify all the signaled jobs in mirror list are removed from the ring + * by waiting for their respective scheduler fences to signal. + * Continually repeat traversing the ring mirror list until no more signaled + * fences are found + */ +retry_wait: spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { if (s_job->s_fence->parent && @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL; atomic_dec(&sched->hw_rq_count); + } else { + wait_fence = dma_fence_get(&s_job->s_fence->finished); + last_job = s_job; + break; } } - spin_unlock_irqrestore(&sched->job_list_lock, flags); - if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { - atomic_inc(&bad->karma); - /* don't increase @bad's karma if it's from KERNEL RQ, - * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs) - * corrupt but keep in mind that kernel jobs always considered good. - */ - for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) { - struct drm_sched_rq *rq = &sched->sched_rq[i]; + /* No signaled jobs in the ring, its safe to proceed to ASIC reset */ + if (!wait_fence) { + spin_unlock_irqrestore(&sched->job_list_lock, flags); + goto done; + } - spin_lock(&rq->lock); - list_for_each_entry_safe(entity, tmp, &rq->entities, list) { - if (bad->s_fence->scheduled.context == entity->fence_context) { - if (atomic_read(&bad->karma) > bad->sched->hang_limit) - if (entity->guilty) - atomic_set(entity->guilty, 1); - break; - } - } - spin_unlock(&rq->lock); - if (&entity->list != &rq->entities) - break; + /* Restore removed cb since removing again already removed cb is undefined */ + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { + if(s_job == last_job) + break;
Need to double check after the holidays, but you should be able to use list_for_each_entry_continue here.
I think it should work - kind of traversing back on all the jobs we just removed their callbacks.
Wrapping your head around stuff again after the holidays sometimes shows new problems we have completely missed :)
Adding the callbacks again won't work because we have freed up our reference to the hardware fence above:
dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL;
We need to drop this or we would never be able to re-add the fences
Yea, that a big miss on my side...
.
But I'm still not sure if we shouldn't rely on the correct order of signaling and simplify this instead.
As you said before, once we switched to signaling the parent from the interrupt context instead of scheduled work no danger of race there, so what if we submit job A and after it job B and B completes before A (like the sync dependency test in libdrm amdgpu tests but without adding explicit dependency to the second command on the first) I believe that still in this case job B's parent (HW) fence will not be signaled before job A completes since EOP event is not signaled until the entire pipe completed and flushed it's cashes including job A. So from this seems to me that indeed it's enough to wait for the last inserted job's parent (HW) fence in ring mirror list to signal. Let me know what you think on that.
P.S V5 is not the last iteration and there was V6 series.
Andrey
Regards, Christian.
Andrey
+ if (s_job->s_fence->parent) { + r = dma_fence_add_callback(s_job->s_fence->parent, + &s_job->s_fence->cb, + drm_sched_process_job); + if (r) + DRM_ERROR("fence restore callback failed (%d)\n", + r);
When you fail to add the callback this means that you need to call call drm_sched_process_job manually here.
} } + spin_unlock_irqrestore(&sched->job_list_lock, flags);
+ dma_fence_wait(wait_fence, false); + dma_fence_put(wait_fence); + wait_fence = NULL;
+ goto retry_wait;
+done: + return;
Drop the done: label and return directly above.
Apart from all those nit picks that starts to look like it should work, Christian.
} -EXPORT_SYMBOL(drm_sched_hw_job_reset); +EXPORT_SYMBOL(drm_sched_stop); /** * drm_sched_job_recovery - recover jobs after a reset @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset); * @sched: scheduler instance * */ -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) { struct drm_sched_job *s_job, *tmp; - bool found_guilty = false; unsigned long flags; int r; + if (!full_recovery) + goto unpark;
- spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
node) { struct drm_sched_fence *s_fence = s_job->s_fence; - struct dma_fence *fence; - uint64_t guilty_context;
- if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { - found_guilty = true; - guilty_context = s_job->s_fence->scheduled.context; - }
- if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) - dma_fence_set_error(&s_fence->finished, -ECANCELED);
- spin_unlock_irqrestore(&sched->job_list_lock, flags); - fence = sched->ops->run_job(s_job); - atomic_inc(&sched->hw_rq_count); + struct dma_fence *fence = s_job->s_fence->parent; if (fence) { - s_fence->parent = dma_fence_get(fence); r = dma_fence_add_callback(fence, &s_fence->cb, drm_sched_process_job); if (r == -ENOENT) @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); - dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(s_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } - spin_lock_irqsave(&sched->job_list_lock, flags); }
- drm_sched_start_timeout(sched); spin_unlock_irqrestore(&sched->job_list_lock, flags);
+unpark: + kthread_unpark(sched->thread); +} +EXPORT_SYMBOL(drm_sched_start);
+/**
- drm_sched_resubmit_jobs - helper to relunch job from mirror ring
list
- @sched: scheduler instance
- */
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_job *s_job, *tmp; + uint64_t guilty_context; + bool found_guilty = false;
+ /*TODO DO we need spinlock here ? */ + list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { + struct drm_sched_fence *s_fence = s_job->s_fence;
+ if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { + found_guilty = true; + guilty_context = s_job->s_fence->scheduled.context; + }
+ if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) + dma_fence_set_error(&s_fence->finished, -ECANCELED);
+ s_job->s_fence->parent = sched->ops->run_job(s_job); + atomic_inc(&sched->hw_rq_count); + } } -EXPORT_SYMBOL(drm_sched_job_recovery); +EXPORT_SYMBOL(drm_sched_resubmit_jobs); /** * drm_sched_job_init - init a scheduler job @@ -634,26 +704,14 @@ static int drm_sched_main(void *param) DRM_ERROR("fence add callback failed (%d)\n", r); dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(sched_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } wake_up(&sched->job_scheduled); } return 0; } -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job) -{ - struct drm_gpu_scheduler *sched = s_job->sched;
- spin_lock(&sched->job_list_lock); - list_del_init(&s_job->node); - spin_unlock(&sched->job_list_lock); -}
- /** * drm_sched_init - Init a gpu scheduler instance *
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 445b2ef..f76d9ed 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job) for (q = 0; q < V3D_MAX_QUEUES; q++) { struct drm_gpu_scheduler *sched = &v3d->queue[q].sched; - kthread_park(sched->thread); - drm_sched_hw_job_reset(sched, (sched_job->sched == sched ? + drm_sched_stop(sched, (sched_job->sched == sched ? sched_job : NULL));
+ if(sched_job) + drm_sched_increase_karma(sched_job); } /* get the GPU back into the init state */ v3d_reset(v3d); + for (q = 0; q < V3D_MAX_QUEUES; q++) + drm_sched_resubmit_jobs(sched_job->sched);
- /* Unblock schedulers and restart their jobs. */ for (q = 0; q < V3D_MAX_QUEUES; q++) {
- drm_sched_job_recovery(&v3d->queue[q].sched); - kthread_unpark(v3d->queue[q].sched.thread); + drm_sched_start(&v3d->queue[q].sched, true); } mutex_unlock(&v3d->reset_lock); diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 47e1979..5ab2d97 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f); * finished to remove the job from the * @drm_gpu_scheduler.ring_mirror_list. * @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list.
- @finish_node: used in a list to wait on before resetting the
scheduler * @id: a unique id assigned to each job scheduled on the scheduler. * @karma: increment on every hang caused by this job. If this exceeds the hang * limit of the scheduler then the job is marked guilty and will not @@ -193,6 +194,7 @@ struct drm_sched_job { struct dma_fence_cb finish_cb; struct work_struct finish_work; struct list_head node; + struct list_head finish_node; uint64_t id; atomic_t karma; enum drm_sched_priority s_priority; @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job, void *owner); void drm_sched_job_cleanup(struct drm_sched_job *job); void drm_sched_wakeup(struct drm_gpu_scheduler *sched); -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, - struct drm_sched_job *job); -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); +void drm_sched_stop(struct drm_gpu_scheduler *sched, + struct drm_sched_job *job); +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); +void drm_sched_increase_karma(struct drm_sched_job *bad); bool drm_sched_dependency_optimized(struct dma_fence* fence, struct drm_sched_entity *entity); void drm_sched_fault(struct drm_gpu_scheduler *sched);
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
On 01/03/2019 03:54 AM, Koenig, Christian wrote:
Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
On 12/21/2018 01:37 PM, Christian König wrote:
Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
Decauple sched threads stop and start and ring mirror list handling from the policy of what to do about the guilty jobs. When stoppping the sched thread and detaching sched fences from non signaled HW fenes wait for all signaled HW fences to complete before rerunning the jobs.
v2: Fix resubmission of guilty job into HW after refactoring.
v4: Full restart for all the jobs, not only from guilty ring. Extract karma increase into standalone function.
v5: Rework waiting for signaled jobs without relying on the job struct itself as those might already be freed for non 'guilty' job's schedulers. Expose karma increase to drivers.
Suggested-by: Christian Koenig Christian.Koenig@amd.com Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- drivers/gpu/drm/scheduler/sched_main.c | 188 +++++++++++++++++++---------- drivers/gpu/drm/v3d/v3d_sched.c | 12 +- include/drm/gpu_scheduler.h | 10 +- 5 files changed, 151 insertions(+), 88 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 8a078f4..a4bd2d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3298,12 +3298,10 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - kthread_park(ring->sched.thread); + drm_sched_stop(&ring->sched, job ? &job->base : NULL); - if (job && job->base.sched != &ring->sched) - continue;
- drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL); + if(job) + drm_sched_increase_karma(&job->base);
Since we dropped the "job && job->base.sched != &ring->sched" check above this will now increase the jobs karma multiple times.
Maybe just move that outside of the loop.
/* after all hw jobs are reset, hw fence is meaningless, so force_completion */ amdgpu_fence_driver_force_completion(ring); @@ -3454,14 +3452,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - /* only need recovery sched of the given job's ring - * or all rings (in the case @job is NULL) - * after above amdgpu_reset accomplished - */ - if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res) - drm_sched_job_recovery(&ring->sched); + if (!adev->asic_reset_res) + drm_sched_resubmit_jobs(&ring->sched); - kthread_unpark(ring->sched.thread); + drm_sched_start(&ring->sched, !adev->asic_reset_res); } if (!amdgpu_device_has_dc_support(adev)) { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 49a6763..6f1268f 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) } /* block scheduler */ - kthread_park(gpu->sched.thread); - drm_sched_hw_job_reset(&gpu->sched, sched_job); + drm_sched_stop(&gpu->sched, sched_job);
+ if(sched_job) + drm_sched_increase_karma(sched_job); /* get the GPU back into the init state */ etnaviv_core_dump(gpu); etnaviv_gpu_recover_hang(gpu); + drm_sched_resubmit_jobs(&gpu->sched);
- /* restart scheduler after GPU is usable again */
- drm_sched_job_recovery(&gpu->sched); - kthread_unpark(gpu->sched.thread); + drm_sched_start(&gpu->sched, true); } static void etnaviv_sched_free_job(struct drm_sched_job *sched_job) diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index dbb6906..b5c5bee 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -60,8 +60,6 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb); -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job);
- /** * drm_sched_rq_init - initialize a given run queue struct *
@@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct work_struct *work) spin_unlock_irqrestore(&sched->job_list_lock, flags); }
Kernel doc here would be nice to have.
+void drm_sched_increase_karma(struct drm_sched_job *bad) +{ + int i; + struct drm_sched_entity *tmp; + struct drm_sched_entity *entity; + struct drm_gpu_scheduler *sched = bad->sched;
+ /* don't increase @bad's karma if it's from KERNEL RQ, + * because sometimes GPU hang would cause kernel jobs (like VM updating jobs) + * corrupt but keep in mind that kernel jobs always considered good. + */ + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; + i++) { + struct drm_sched_rq *rq = &sched->sched_rq[i];
+ spin_lock(&rq->lock); + list_for_each_entry_safe(entity, tmp, &rq->entities, list) { + if (bad->s_fence->scheduled.context == + entity->fence_context) { + if (atomic_read(&bad->karma) > + bad->sched->hang_limit) + if (entity->guilty) + atomic_set(entity->guilty, 1); + break; + } + } + spin_unlock(&rq->lock); + if (&entity->list != &rq->entities) + break; + } + } +} +EXPORT_SYMBOL(drm_sched_increase_karma);
- /** * drm_sched_hw_job_reset - stop the scheduler if it contains the
bad job * @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct work_struct *work) * @bad: bad scheduler job * */ -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) { - struct drm_sched_job *s_job; - struct drm_sched_entity *entity, *tmp; + struct drm_sched_job *s_job, *last_job; unsigned long flags; - int i; + struct dma_fence *wait_fence = NULL; + int r;
+ kthread_park(sched->thread); + /* + * Verify all the signaled jobs in mirror list are removed from the ring + * by waiting for their respective scheduler fences to signal. + * Continually repeat traversing the ring mirror list until no more signaled + * fences are found + */ +retry_wait: spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { if (s_job->s_fence->parent && @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL; atomic_dec(&sched->hw_rq_count); + } else { + wait_fence = dma_fence_get(&s_job->s_fence->finished); + last_job = s_job; + break; } } - spin_unlock_irqrestore(&sched->job_list_lock, flags); - if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { - atomic_inc(&bad->karma); - /* don't increase @bad's karma if it's from KERNEL RQ, - * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs) - * corrupt but keep in mind that kernel jobs always considered good. - */ - for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) { - struct drm_sched_rq *rq = &sched->sched_rq[i]; + /* No signaled jobs in the ring, its safe to proceed to ASIC reset */ + if (!wait_fence) { + spin_unlock_irqrestore(&sched->job_list_lock, flags); + goto done; + } - spin_lock(&rq->lock); - list_for_each_entry_safe(entity, tmp, &rq->entities, list) { - if (bad->s_fence->scheduled.context == entity->fence_context) { - if (atomic_read(&bad->karma) > bad->sched->hang_limit) - if (entity->guilty) - atomic_set(entity->guilty, 1); - break; - } - } - spin_unlock(&rq->lock); - if (&entity->list != &rq->entities) - break; + /* Restore removed cb since removing again already removed cb is undefined */ + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { + if(s_job == last_job) + break;
Need to double check after the holidays, but you should be able to use list_for_each_entry_continue here.
I think it should work - kind of traversing back on all the jobs we just removed their callbacks.
Wrapping your head around stuff again after the holidays sometimes shows new problems we have completely missed :)
Adding the callbacks again won't work because we have freed up our reference to the hardware fence above:
dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL;
We need to drop this or we would never be able to re-add the fences
Yea, that a big miss on my side...
.
But I'm still not sure if we shouldn't rely on the correct order of signaling and simplify this instead.
As you said before, once we switched to signaling the parent from the interrupt context instead of scheduled work no danger of race there
Correction here - once we switched removing the job from mirror_ring list directly in interrupt context instead later from scheduled work
Andrey
, so what if we submit job A and after it job B and B completes before A (like the sync dependency test in libdrm amdgpu tests but without adding explicit dependency to the second command on the first) I believe that still in this case job B's parent (HW) fence will not be signaled before job A completes since EOP event is not signaled until the entire pipe completed and flushed it's cashes including job A. So from this seems to me that indeed it's enough to wait for the last inserted job's parent (HW) fence in ring mirror list to signal. Let me know what you think on that.
P.S V5 is not the last iteration and there was V6 series.
Andrey
Regards, Christian.
Andrey
+ if (s_job->s_fence->parent) { + r = dma_fence_add_callback(s_job->s_fence->parent, + &s_job->s_fence->cb, + drm_sched_process_job); + if (r) + DRM_ERROR("fence restore callback failed (%d)\n", + r);
When you fail to add the callback this means that you need to call call drm_sched_process_job manually here.
} } + spin_unlock_irqrestore(&sched->job_list_lock, flags);
+ dma_fence_wait(wait_fence, false); + dma_fence_put(wait_fence); + wait_fence = NULL;
+ goto retry_wait;
+done: + return;
Drop the done: label and return directly above.
Apart from all those nit picks that starts to look like it should work, Christian.
} -EXPORT_SYMBOL(drm_sched_hw_job_reset); +EXPORT_SYMBOL(drm_sched_stop); /** * drm_sched_job_recovery - recover jobs after a reset @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset); * @sched: scheduler instance * */ -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) { struct drm_sched_job *s_job, *tmp; - bool found_guilty = false; unsigned long flags; int r; + if (!full_recovery) + goto unpark;
- spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
node) { struct drm_sched_fence *s_fence = s_job->s_fence; - struct dma_fence *fence; - uint64_t guilty_context;
- if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { - found_guilty = true; - guilty_context = s_job->s_fence->scheduled.context; - }
- if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) - dma_fence_set_error(&s_fence->finished, -ECANCELED);
- spin_unlock_irqrestore(&sched->job_list_lock, flags); - fence = sched->ops->run_job(s_job); - atomic_inc(&sched->hw_rq_count); + struct dma_fence *fence = s_job->s_fence->parent; if (fence) { - s_fence->parent = dma_fence_get(fence); r = dma_fence_add_callback(fence, &s_fence->cb, drm_sched_process_job); if (r == -ENOENT) @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); - dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(s_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } - spin_lock_irqsave(&sched->job_list_lock, flags); }
- drm_sched_start_timeout(sched); spin_unlock_irqrestore(&sched->job_list_lock, flags);
+unpark: + kthread_unpark(sched->thread); +} +EXPORT_SYMBOL(drm_sched_start);
+/**
- drm_sched_resubmit_jobs - helper to relunch job from mirror ring
list
- @sched: scheduler instance
- */
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_job *s_job, *tmp; + uint64_t guilty_context; + bool found_guilty = false;
+ /*TODO DO we need spinlock here ? */ + list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { + struct drm_sched_fence *s_fence = s_job->s_fence;
+ if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { + found_guilty = true; + guilty_context = s_job->s_fence->scheduled.context; + }
+ if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) + dma_fence_set_error(&s_fence->finished, -ECANCELED);
+ s_job->s_fence->parent = sched->ops->run_job(s_job); + atomic_inc(&sched->hw_rq_count); + } } -EXPORT_SYMBOL(drm_sched_job_recovery); +EXPORT_SYMBOL(drm_sched_resubmit_jobs); /** * drm_sched_job_init - init a scheduler job @@ -634,26 +704,14 @@ static int drm_sched_main(void *param) DRM_ERROR("fence add callback failed (%d)\n", r); dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(sched_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } wake_up(&sched->job_scheduled); } return 0; } -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job) -{ - struct drm_gpu_scheduler *sched = s_job->sched;
- spin_lock(&sched->job_list_lock); - list_del_init(&s_job->node); - spin_unlock(&sched->job_list_lock); -}
- /** * drm_sched_init - Init a gpu scheduler instance *
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 445b2ef..f76d9ed 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job) for (q = 0; q < V3D_MAX_QUEUES; q++) { struct drm_gpu_scheduler *sched = &v3d->queue[q].sched; - kthread_park(sched->thread); - drm_sched_hw_job_reset(sched, (sched_job->sched == sched ? + drm_sched_stop(sched, (sched_job->sched == sched ? sched_job : NULL));
+ if(sched_job) + drm_sched_increase_karma(sched_job); } /* get the GPU back into the init state */ v3d_reset(v3d); + for (q = 0; q < V3D_MAX_QUEUES; q++) + drm_sched_resubmit_jobs(sched_job->sched);
- /* Unblock schedulers and restart their jobs. */ for (q = 0; q < V3D_MAX_QUEUES; q++) {
- drm_sched_job_recovery(&v3d->queue[q].sched); - kthread_unpark(v3d->queue[q].sched.thread); + drm_sched_start(&v3d->queue[q].sched, true); } mutex_unlock(&v3d->reset_lock); diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 47e1979..5ab2d97 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f); * finished to remove the job from the * @drm_gpu_scheduler.ring_mirror_list. * @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list.
- @finish_node: used in a list to wait on before resetting the
scheduler * @id: a unique id assigned to each job scheduled on the scheduler. * @karma: increment on every hang caused by this job. If this exceeds the hang * limit of the scheduler then the job is marked guilty and will not @@ -193,6 +194,7 @@ struct drm_sched_job { struct dma_fence_cb finish_cb; struct work_struct finish_work; struct list_head node; + struct list_head finish_node; uint64_t id; atomic_t karma; enum drm_sched_priority s_priority; @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job, void *owner); void drm_sched_job_cleanup(struct drm_sched_job *job); void drm_sched_wakeup(struct drm_gpu_scheduler *sched); -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, - struct drm_sched_job *job); -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); +void drm_sched_stop(struct drm_gpu_scheduler *sched, + struct drm_sched_job *job); +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); +void drm_sched_increase_karma(struct drm_sched_job *bad); bool drm_sched_dependency_optimized(struct dma_fence* fence, struct drm_sched_entity *entity); void drm_sched_fault(struct drm_gpu_scheduler *sched);
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
Am 03.01.19 um 18:42 schrieb Grodzovsky, Andrey:
On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
On 01/03/2019 03:54 AM, Koenig, Christian wrote:
Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
On 12/21/2018 01:37 PM, Christian König wrote:
Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky:
Decauple sched threads stop and start and ring mirror list handling from the policy of what to do about the guilty jobs. When stoppping the sched thread and detaching sched fences from non signaled HW fenes wait for all signaled HW fences to complete before rerunning the jobs.
v2: Fix resubmission of guilty job into HW after refactoring.
v4: Full restart for all the jobs, not only from guilty ring. Extract karma increase into standalone function.
v5: Rework waiting for signaled jobs without relying on the job struct itself as those might already be freed for non 'guilty' job's schedulers. Expose karma increase to drivers.
Suggested-by: Christian Koenig Christian.Koenig@amd.com Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- drivers/gpu/drm/scheduler/sched_main.c | 188+++++++++++++++++++---------- drivers/gpu/drm/v3d/v3d_sched.c | 12 +- include/drm/gpu_scheduler.h | 10 +- 5 files changed, 151 insertions(+), 88 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 8a078f4..a4bd2d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3298,12 +3298,10 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - kthread_park(ring->sched.thread); + drm_sched_stop(&ring->sched, job ? &job->base : NULL); - if (job && job->base.sched != &ring->sched) - continue;
- drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL); + if(job) + drm_sched_increase_karma(&job->base);
Since we dropped the "job && job->base.sched != &ring->sched" check above this will now increase the jobs karma multiple times.
Maybe just move that outside of the loop.
/* after all hw jobs are reset, hw fence is meaningless,so force_completion */ amdgpu_fence_driver_force_completion(ring); @@ -3454,14 +3452,10 @@ static void amdgpu_device_post_asic_reset(struct amdgpu_device *adev, if (!ring || !ring->sched.thread) continue; - /* only need recovery sched of the given job's ring - * or all rings (in the case @job is NULL) - * after above amdgpu_reset accomplished - */ - if ((!job || job->base.sched == &ring->sched) && !adev->asic_reset_res) - drm_sched_job_recovery(&ring->sched); + if (!adev->asic_reset_res) + drm_sched_resubmit_jobs(&ring->sched); - kthread_unpark(ring->sched.thread); + drm_sched_start(&ring->sched, !adev->asic_reset_res); } if (!amdgpu_device_has_dc_support(adev)) { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 49a6763..6f1268f 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) } /* block scheduler */ - kthread_park(gpu->sched.thread); - drm_sched_hw_job_reset(&gpu->sched, sched_job); + drm_sched_stop(&gpu->sched, sched_job);
+ if(sched_job) + drm_sched_increase_karma(sched_job); /* get the GPU back into the init state */ etnaviv_core_dump(gpu); etnaviv_gpu_recover_hang(gpu); + drm_sched_resubmit_jobs(&gpu->sched);
- /* restart scheduler after GPU is usable again */
- drm_sched_job_recovery(&gpu->sched); - kthread_unpark(gpu->sched.thread); + drm_sched_start(&gpu->sched, true); } static void etnaviv_sched_free_job(struct drm_sched_job *sched_job) diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index dbb6906..b5c5bee 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -60,8 +60,6 @@ static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb); -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job);
- /** * drm_sched_rq_init - initialize a given run queue struct *
@@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct work_struct *work) spin_unlock_irqrestore(&sched->job_list_lock, flags); }
Kernel doc here would be nice to have.
+void drm_sched_increase_karma(struct drm_sched_job *bad) +{ + int i; + struct drm_sched_entity *tmp; + struct drm_sched_entity *entity; + struct drm_gpu_scheduler *sched = bad->sched;
+ /* don't increase @bad's karma if it's from KERNEL RQ, + * because sometimes GPU hang would cause kernel jobs (like VM updating jobs) + * corrupt but keep in mind that kernel jobs always considered good. + */ + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; + i++) { + struct drm_sched_rq *rq = &sched->sched_rq[i];
+ spin_lock(&rq->lock); + list_for_each_entry_safe(entity, tmp, &rq->entities, list) { + if (bad->s_fence->scheduled.context == + entity->fence_context) { + if (atomic_read(&bad->karma) > + bad->sched->hang_limit) + if (entity->guilty) + atomic_set(entity->guilty, 1); + break; + } + } + spin_unlock(&rq->lock); + if (&entity->list != &rq->entities) + break; + } + } +} +EXPORT_SYMBOL(drm_sched_increase_karma);
- /** * drm_sched_hw_job_reset - stop the scheduler if it contains the
bad job * @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct work_struct *work) * @bad: bad scheduler job * */ -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) { - struct drm_sched_job *s_job; - struct drm_sched_entity *entity, *tmp; + struct drm_sched_job *s_job, *last_job; unsigned long flags; - int i; + struct dma_fence *wait_fence = NULL; + int r;
+ kthread_park(sched->thread); + /* + * Verify all the signaled jobs in mirror list are removed from the ring + * by waiting for their respective scheduler fences to signal. + * Continually repeat traversing the ring mirror list until no more signaled + * fences are found + */ +retry_wait: spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { if (s_job->s_fence->parent && @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_jo dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL; atomic_dec(&sched->hw_rq_count); + } else { + wait_fence = dma_fence_get(&s_job->s_fence->finished); + last_job = s_job; + break; } } - spin_unlock_irqrestore(&sched->job_list_lock, flags); - if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { - atomic_inc(&bad->karma); - /* don't increase @bad's karma if it's from KERNEL RQ, - * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs) - * corrupt but keep in mind that kernel jobs always considered good. - */ - for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) { - struct drm_sched_rq *rq = &sched->sched_rq[i]; + /* No signaled jobs in the ring, its safe to proceed to ASIC reset */ + if (!wait_fence) { + spin_unlock_irqrestore(&sched->job_list_lock, flags); + goto done; + } - spin_lock(&rq->lock); - list_for_each_entry_safe(entity, tmp, &rq->entities, list) { - if (bad->s_fence->scheduled.context == entity->fence_context) { - if (atomic_read(&bad->karma) > bad->sched->hang_limit) - if (entity->guilty) - atomic_set(entity->guilty, 1); - break; - } - } - spin_unlock(&rq->lock); - if (&entity->list != &rq->entities) - break; + /* Restore removed cb since removing again already removed cb is undefined */ + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { + if(s_job == last_job) + break;
Need to double check after the holidays, but you should be able to use list_for_each_entry_continue here.
I think it should work - kind of traversing back on all the jobs we just removed their callbacks.
Wrapping your head around stuff again after the holidays sometimes shows new problems we have completely missed :)
Adding the callbacks again won't work because we have freed up our reference to the hardware fence above:
dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL;We need to drop this or we would never be able to re-add the fences
Yea, that a big miss on my side...
.
But I'm still not sure if we shouldn't rely on the correct order of signaling and simplify this instead.
As you said before, once we switched to signaling the parent from the interrupt context instead of scheduled work no danger of race there
Correction here - once we switched removing the job from mirror_ring list directly in interrupt context instead later from scheduled work
Ok, so let's stick with the approach of only waiting for the first signaled one found.
But we need to remove setting the parent fence to NULL or otherwise we won't be able to add the callback ever again.
Christian.
Andrey
, so what if we submit job A and after it job B and B completes before A (like the sync dependency test in libdrm amdgpu tests but without adding explicit dependency to the second command on the first) I believe that still in this case job B's parent (HW) fence will not be signaled before job A completes since EOP event is not signaled until the entire pipe completed and flushed it's cashes including job A. So from this seems to me that indeed it's enough to wait for the last inserted job's parent (HW) fence in ring mirror list to signal. Let me know what you think on that.
P.S V5 is not the last iteration and there was V6 series.
Andrey
Regards, Christian.
Andrey
+ if (s_job->s_fence->parent) { + r = dma_fence_add_callback(s_job->s_fence->parent, + &s_job->s_fence->cb, + drm_sched_process_job); + if (r) + DRM_ERROR("fence restore callback failed (%d)\n", + r);
When you fail to add the callback this means that you need to call call drm_sched_process_job manually here.
} }+ spin_unlock_irqrestore(&sched->job_list_lock, flags);
+ dma_fence_wait(wait_fence, false); + dma_fence_put(wait_fence); + wait_fence = NULL;
+ goto retry_wait;
+done: + return;
Drop the done: label and return directly above.
Apart from all those nit picks that starts to look like it should work, Christian.
}-EXPORT_SYMBOL(drm_sched_hw_job_reset); +EXPORT_SYMBOL(drm_sched_stop); /** * drm_sched_job_recovery - recover jobs after a reset @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset); * @sched: scheduler instance * */ -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) { struct drm_sched_job *s_job, *tmp; - bool found_guilty = false; unsigned long flags; int r; + if (!full_recovery) + goto unpark;
- spin_lock_irqsave(&sched->job_list_lock, flags); list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list,
node) { struct drm_sched_fence *s_fence = s_job->s_fence; - struct dma_fence *fence; - uint64_t guilty_context;
- if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { - found_guilty = true; - guilty_context = s_job->s_fence->scheduled.context; - }
- if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) - dma_fence_set_error(&s_fence->finished, -ECANCELED);
- spin_unlock_irqrestore(&sched->job_list_lock, flags); - fence = sched->ops->run_job(s_job); - atomic_inc(&sched->hw_rq_count); + struct dma_fence *fence = s_job->s_fence->parent; if (fence) { - s_fence->parent = dma_fence_get(fence); r = dma_fence_add_callback(fence, &s_fence->cb, drm_sched_process_job); if (r == -ENOENT) @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); - dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(s_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } - spin_lock_irqsave(&sched->job_list_lock, flags); }
- drm_sched_start_timeout(sched); spin_unlock_irqrestore(&sched->job_list_lock, flags);
+unpark: + kthread_unpark(sched->thread); +} +EXPORT_SYMBOL(drm_sched_start);
+/**
- drm_sched_resubmit_jobs - helper to relunch job from mirror ring
list
- @sched: scheduler instance
- */
+void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) +{ + struct drm_sched_job *s_job, *tmp; + uint64_t guilty_context; + bool found_guilty = false;
+ /*TODO DO we need spinlock here ? */ + list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { + struct drm_sched_fence *s_fence = s_job->s_fence;
+ if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { + found_guilty = true; + guilty_context = s_job->s_fence->scheduled.context; + }
+ if (found_guilty && s_job->s_fence->scheduled.context == guilty_context) + dma_fence_set_error(&s_fence->finished, -ECANCELED);
+ s_job->s_fence->parent = sched->ops->run_job(s_job); + atomic_inc(&sched->hw_rq_count); + } } -EXPORT_SYMBOL(drm_sched_job_recovery); +EXPORT_SYMBOL(drm_sched_resubmit_jobs); /** * drm_sched_job_init - init a scheduler job @@ -634,26 +704,14 @@ static int drm_sched_main(void *param) DRM_ERROR("fence add callback failed (%d)\n", r); dma_fence_put(fence); - } else { - if (s_fence->finished.error < 0) - drm_sched_expel_job_unlocked(sched_job); + } else drm_sched_process_job(NULL, &s_fence->cb); - } wake_up(&sched->job_scheduled); } return 0; } -static void drm_sched_expel_job_unlocked(struct drm_sched_job *s_job) -{ - struct drm_gpu_scheduler *sched = s_job->sched;
- spin_lock(&sched->job_list_lock); - list_del_init(&s_job->node); - spin_unlock(&sched->job_list_lock); -}
- /** * drm_sched_init - Init a gpu scheduler instance *
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 445b2ef..f76d9ed 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job *sched_job) for (q = 0; q < V3D_MAX_QUEUES; q++) { struct drm_gpu_scheduler *sched = &v3d->queue[q].sched; - kthread_park(sched->thread); - drm_sched_hw_job_reset(sched, (sched_job->sched == sched ? + drm_sched_stop(sched, (sched_job->sched == sched ? sched_job : NULL));
+ if(sched_job) + drm_sched_increase_karma(sched_job); } /* get the GPU back into the init state */ v3d_reset(v3d); + for (q = 0; q < V3D_MAX_QUEUES; q++) + drm_sched_resubmit_jobs(sched_job->sched);
- /* Unblock schedulers and restart their jobs. */ for (q = 0; q < V3D_MAX_QUEUES; q++) {
- drm_sched_job_recovery(&v3d->queue[q].sched); - kthread_unpark(v3d->queue[q].sched.thread); + drm_sched_start(&v3d->queue[q].sched, true); } mutex_unlock(&v3d->reset_lock); diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 47e1979..5ab2d97 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -175,6 +175,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f); * finished to remove the job from the * @drm_gpu_scheduler.ring_mirror_list. * @node: used to append this struct to the @drm_gpu_scheduler.ring_mirror_list.
- @finish_node: used in a list to wait on before resetting the
scheduler * @id: a unique id assigned to each job scheduled on the scheduler. * @karma: increment on every hang caused by this job. If this exceeds the hang * limit of the scheduler then the job is marked guilty and will not @@ -193,6 +194,7 @@ struct drm_sched_job { struct dma_fence_cb finish_cb; struct work_struct finish_work; struct list_head node; + struct list_head finish_node; uint64_t id; atomic_t karma; enum drm_sched_priority s_priority; @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job *job, void *owner); void drm_sched_job_cleanup(struct drm_sched_job *job); void drm_sched_wakeup(struct drm_gpu_scheduler *sched); -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, - struct drm_sched_job *job); -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); +void drm_sched_stop(struct drm_gpu_scheduler *sched, + struct drm_sched_job *job); +void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); +void drm_sched_increase_karma(struct drm_sched_job *bad); bool drm_sched_dependency_optimized(struct dma_fence* fence, struct drm_sched_entity *entity); void drm_sched_fault(struct drm_gpu_scheduler *sched);
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
On 01/07/2019 09:13 AM, Christian König wrote:
Am 03.01.19 um 18:42 schrieb Grodzovsky, Andrey:
On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
On 01/03/2019 03:54 AM, Koenig, Christian wrote:
Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
On 12/21/2018 01:37 PM, Christian König wrote:
Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky: > Decauple sched threads stop and start and ring mirror > list handling from the policy of what to do about the > guilty jobs. > When stoppping the sched thread and detaching sched fences > from non signaled HW fenes wait for all signaled HW fences > to complete before rerunning the jobs. > > v2: Fix resubmission of guilty job into HW after refactoring. > > v4: > Full restart for all the jobs, not only from guilty ring. > Extract karma increase into standalone function. > > v5: > Rework waiting for signaled jobs without relying on the job > struct itself as those might already be freed for non 'guilty' > job's schedulers. > Expose karma increase to drivers. > > Suggested-by: Christian Koenig Christian.Koenig@amd.com > Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com > --- > drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- > drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- > drivers/gpu/drm/scheduler/sched_main.c | 188 > +++++++++++++++++++---------- > drivers/gpu/drm/v3d/v3d_sched.c | 12 +- > include/drm/gpu_scheduler.h | 10 +- > 5 files changed, 151 insertions(+), 88 deletions(-) > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > index 8a078f4..a4bd2d3 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c > @@ -3298,12 +3298,10 @@ static int > amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, > if (!ring || !ring->sched.thread) > continue; > - kthread_park(ring->sched.thread); > + drm_sched_stop(&ring->sched, job ? &job->base : NULL); > - if (job && job->base.sched != &ring->sched) > - continue; > - > - drm_sched_hw_job_reset(&ring->sched, job ? &job->base : > NULL); > + if(job) > + drm_sched_increase_karma(&job->base); Since we dropped the "job && job->base.sched != &ring->sched" check above this will now increase the jobs karma multiple times.
Maybe just move that outside of the loop.
> /* after all hw jobs are reset, hw fence is > meaningless, > so force_completion */ > amdgpu_fence_driver_force_completion(ring); > @@ -3454,14 +3452,10 @@ static void > amdgpu_device_post_asic_reset(struct amdgpu_device *adev, > if (!ring || !ring->sched.thread) > continue; > - /* only need recovery sched of the given job's ring > - * or all rings (in the case @job is NULL) > - * after above amdgpu_reset accomplished > - */ > - if ((!job || job->base.sched == &ring->sched) && > !adev->asic_reset_res) > - drm_sched_job_recovery(&ring->sched); > + if (!adev->asic_reset_res) > + drm_sched_resubmit_jobs(&ring->sched); > - kthread_unpark(ring->sched.thread); > + drm_sched_start(&ring->sched, !adev->asic_reset_res); > } > if (!amdgpu_device_has_dc_support(adev)) { > diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c > b/drivers/gpu/drm/etnaviv/etnaviv_sched.c > index 49a6763..6f1268f 100644 > --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c > +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c > @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct > drm_sched_job *sched_job) > } > /* block scheduler */ > - kthread_park(gpu->sched.thread); > - drm_sched_hw_job_reset(&gpu->sched, sched_job); > + drm_sched_stop(&gpu->sched, sched_job); > + > + if(sched_job) > + drm_sched_increase_karma(sched_job); > /* get the GPU back into the init state */ > etnaviv_core_dump(gpu); > etnaviv_gpu_recover_hang(gpu); > + drm_sched_resubmit_jobs(&gpu->sched); > + > /* restart scheduler after GPU is usable again */ > - drm_sched_job_recovery(&gpu->sched); > - kthread_unpark(gpu->sched.thread); > + drm_sched_start(&gpu->sched, true); > } > static void etnaviv_sched_free_job(struct drm_sched_job > *sched_job) > diff --git a/drivers/gpu/drm/scheduler/sched_main.c > b/drivers/gpu/drm/scheduler/sched_main.c > index dbb6906..b5c5bee 100644 > --- a/drivers/gpu/drm/scheduler/sched_main.c > +++ b/drivers/gpu/drm/scheduler/sched_main.c > @@ -60,8 +60,6 @@ > static void drm_sched_process_job(struct dma_fence *f, > struct > dma_fence_cb *cb); > -static void drm_sched_expel_job_unlocked(struct > drm_sched_job > *s_job); > - > /** > * drm_sched_rq_init - initialize a given run queue struct > * > @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct > work_struct *work) > spin_unlock_irqrestore(&sched->job_list_lock, flags); > } Kernel doc here would be nice to have.
> +void drm_sched_increase_karma(struct drm_sched_job *bad) > +{ > + int i; > + struct drm_sched_entity *tmp; > + struct drm_sched_entity *entity; > + struct drm_gpu_scheduler *sched = bad->sched; > + > + /* don't increase @bad's karma if it's from KERNEL RQ, > + * because sometimes GPU hang would cause kernel jobs (like VM > updating jobs) > + * corrupt but keep in mind that kernel jobs always considered > good. > + */ > + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { > + atomic_inc(&bad->karma); > + for (i = DRM_SCHED_PRIORITY_MIN; i < > DRM_SCHED_PRIORITY_KERNEL; > + i++) { > + struct drm_sched_rq *rq = &sched->sched_rq[i]; > + > + spin_lock(&rq->lock); > + list_for_each_entry_safe(entity, tmp, &rq->entities, > list) { > + if (bad->s_fence->scheduled.context == > + entity->fence_context) { > + if (atomic_read(&bad->karma) > > + bad->sched->hang_limit) > + if (entity->guilty) > + atomic_set(entity->guilty, 1); > + break; > + } > + } > + spin_unlock(&rq->lock); > + if (&entity->list != &rq->entities) > + break; > + } > + } > +} > +EXPORT_SYMBOL(drm_sched_increase_karma); > + > /** > * drm_sched_hw_job_reset - stop the scheduler if it > contains the > bad job > * > @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct > work_struct *work) > * @bad: bad scheduler job > * > */ > -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, > struct > drm_sched_job *bad) > +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct > drm_sched_job *bad) > { > - struct drm_sched_job *s_job; > - struct drm_sched_entity *entity, *tmp; > + struct drm_sched_job *s_job, *last_job; > unsigned long flags; > - int i; > + struct dma_fence *wait_fence = NULL; > + int r; > + > + kthread_park(sched->thread); > + /* > + * Verify all the signaled jobs in mirror list are removed > from > the ring > + * by waiting for their respective scheduler fences to signal. > + * Continually repeat traversing the ring mirror list > until no > more signaled > + * fences are found > + */ > +retry_wait: > spin_lock_irqsave(&sched->job_list_lock, flags); > list_for_each_entry_reverse(s_job, > &sched->ring_mirror_list, > node) { > if (s_job->s_fence->parent && > @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct > drm_gpu_scheduler *sched, struct drm_sched_jo > dma_fence_put(s_job->s_fence->parent); > s_job->s_fence->parent = NULL; > atomic_dec(&sched->hw_rq_count); > + } else { > + wait_fence = > dma_fence_get(&s_job->s_fence->finished); > + last_job = s_job; > + break; > } > } > - spin_unlock_irqrestore(&sched->job_list_lock, flags); > - if (bad && bad->s_priority != > DRM_SCHED_PRIORITY_KERNEL) { > - atomic_inc(&bad->karma); > - /* don't increase @bad's karma if it's from KERNEL RQ, > - * becuase sometimes GPU hang would cause kernel jobs > (like > VM updating jobs) > - * corrupt but keep in mind that kernel jobs always > considered good. > - */ > - for (i = DRM_SCHED_PRIORITY_MIN; i < > DRM_SCHED_PRIORITY_KERNEL; i++ ) { > - struct drm_sched_rq *rq = &sched->sched_rq[i]; > + /* No signaled jobs in the ring, its safe to proceed to ASIC > reset */ > + if (!wait_fence) { > + spin_unlock_irqrestore(&sched->job_list_lock, flags); > + goto done; > + } > - spin_lock(&rq->lock); > - list_for_each_entry_safe(entity, tmp, &rq->entities, > list) { > - if (bad->s_fence->scheduled.context == > entity->fence_context) { > - if (atomic_read(&bad->karma) > > bad->sched->hang_limit) > - if (entity->guilty) > - atomic_set(entity->guilty, 1); > - break; > - } > - } > - spin_unlock(&rq->lock); > - if (&entity->list != &rq->entities) > - break; > + /* Restore removed cb since removing again already removed > cb is > undefined */ > + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, > node) { > + if(s_job == last_job) > + break; Need to double check after the holidays, but you should be able to use list_for_each_entry_continue here.
I think it should work - kind of traversing back on all the jobs we just removed their callbacks.
Wrapping your head around stuff again after the holidays sometimes shows new problems we have completely missed :)
Adding the callbacks again won't work because we have freed up our reference to the hardware fence above:
dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL;
We need to drop this or we would never be able to re-add the fences
Yea, that a big miss on my side...
.
But I'm still not sure if we shouldn't rely on the correct order of signaling and simplify this instead.
As you said before, once we switched to signaling the parent from the interrupt context instead of scheduled work no danger of race there
Correction here - once we switched removing the job from mirror_ring list directly in interrupt context instead later from scheduled work
Ok, so let's stick with the approach of only waiting for the first signaled one found.
But we need to remove setting the parent fence to NULL or otherwise we won't be able to add the callback ever again.
Christian.
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Andrey
Andrey
, so what if we submit job A and after it job B and B completes before A (like the sync dependency test in libdrm amdgpu tests but without adding explicit dependency to the second command on the first) I believe that still in this case job B's parent (HW) fence will not be signaled before job A completes since EOP event is not signaled until the entire pipe completed and flushed it's cashes including job A. So from this seems to me that indeed it's enough to wait for the last inserted job's parent (HW) fence in ring mirror list to signal. Let me know what you think on that.
P.S V5 is not the last iteration and there was V6 series.
Andrey
Regards, Christian.
Andrey
> + > + if (s_job->s_fence->parent) { > + r = dma_fence_add_callback(s_job->s_fence->parent, > + &s_job->s_fence->cb, > + drm_sched_process_job); > + if (r) > + DRM_ERROR("fence restore callback failed (%d)\n", > + r); When you fail to add the callback this means that you need to call call drm_sched_process_job manually here.
> } > } > + spin_unlock_irqrestore(&sched->job_list_lock, flags); > + > + dma_fence_wait(wait_fence, false); > + dma_fence_put(wait_fence); > + wait_fence = NULL; > + > + goto retry_wait; > + > +done: > + return; Drop the done: label and return directly above.
Apart from all those nit picks that starts to look like it should work, Christian.
> } > -EXPORT_SYMBOL(drm_sched_hw_job_reset); > +EXPORT_SYMBOL(drm_sched_stop); > /** > * drm_sched_job_recovery - recover jobs after a reset > @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset); > * @sched: scheduler instance > * > */ > -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) > +void drm_sched_start(struct drm_gpu_scheduler *sched, bool > full_recovery) > { > struct drm_sched_job *s_job, *tmp; > - bool found_guilty = false; > unsigned long flags; > int r; > + if (!full_recovery) > + goto unpark; > + > spin_lock_irqsave(&sched->job_list_lock, flags); > list_for_each_entry_safe(s_job, tmp, > &sched->ring_mirror_list, > node) { > struct drm_sched_fence *s_fence = s_job->s_fence; > - struct dma_fence *fence; > - uint64_t guilty_context; > - > - if (!found_guilty && atomic_read(&s_job->karma) > > sched->hang_limit) { > - found_guilty = true; > - guilty_context = s_job->s_fence->scheduled.context; > - } > - > - if (found_guilty && s_job->s_fence->scheduled.context == > guilty_context) > - dma_fence_set_error(&s_fence->finished, -ECANCELED); > - > - spin_unlock_irqrestore(&sched->job_list_lock, flags); > - fence = sched->ops->run_job(s_job); > - atomic_inc(&sched->hw_rq_count); > + struct dma_fence *fence = s_job->s_fence->parent; > if (fence) { > - s_fence->parent = dma_fence_get(fence); > r = dma_fence_add_callback(fence, &s_fence->cb, > drm_sched_process_job); > if (r == -ENOENT) > @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct > drm_gpu_scheduler *sched) > else if (r) > DRM_ERROR("fence add callback failed (%d)\n", > r); > - dma_fence_put(fence); > - } else { > - if (s_fence->finished.error < 0) > - drm_sched_expel_job_unlocked(s_job); > + } else > drm_sched_process_job(NULL, &s_fence->cb); > - } > - spin_lock_irqsave(&sched->job_list_lock, flags); > } > + > drm_sched_start_timeout(sched); > spin_unlock_irqrestore(&sched->job_list_lock, flags); > + > +unpark: > + kthread_unpark(sched->thread); > +} > +EXPORT_SYMBOL(drm_sched_start); > + > +/** > + * drm_sched_resubmit_jobs - helper to relunch job from mirror > ring > list > + * > + * @sched: scheduler instance > + * > + */ > +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) > +{ > + struct drm_sched_job *s_job, *tmp; > + uint64_t guilty_context; > + bool found_guilty = false; > + > + /*TODO DO we need spinlock here ? */ > + list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, > node) { > + struct drm_sched_fence *s_fence = s_job->s_fence; > + > + if (!found_guilty && atomic_read(&s_job->karma) > > sched->hang_limit) { > + found_guilty = true; > + guilty_context = s_job->s_fence->scheduled.context; > + } > + > + if (found_guilty && s_job->s_fence->scheduled.context == > guilty_context) > + dma_fence_set_error(&s_fence->finished, -ECANCELED); > + > + s_job->s_fence->parent = sched->ops->run_job(s_job); > + atomic_inc(&sched->hw_rq_count); > + } > } > -EXPORT_SYMBOL(drm_sched_job_recovery); > +EXPORT_SYMBOL(drm_sched_resubmit_jobs); > /** > * drm_sched_job_init - init a scheduler job > @@ -634,26 +704,14 @@ static int drm_sched_main(void *param) > DRM_ERROR("fence add callback failed (%d)\n", > r); > dma_fence_put(fence); > - } else { > - if (s_fence->finished.error < 0) > - drm_sched_expel_job_unlocked(sched_job); > + } else > drm_sched_process_job(NULL, &s_fence->cb); > - } > wake_up(&sched->job_scheduled); > } > return 0; > } > -static void drm_sched_expel_job_unlocked(struct > drm_sched_job *s_job) > -{ > - struct drm_gpu_scheduler *sched = s_job->sched; > - > - spin_lock(&sched->job_list_lock); > - list_del_init(&s_job->node); > - spin_unlock(&sched->job_list_lock); > -} > - > /** > * drm_sched_init - Init a gpu scheduler instance > * > diff --git a/drivers/gpu/drm/v3d/v3d_sched.c > b/drivers/gpu/drm/v3d/v3d_sched.c > index 445b2ef..f76d9ed 100644 > --- a/drivers/gpu/drm/v3d/v3d_sched.c > +++ b/drivers/gpu/drm/v3d/v3d_sched.c > @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job > *sched_job) > for (q = 0; q < V3D_MAX_QUEUES; q++) { > struct drm_gpu_scheduler *sched = > &v3d->queue[q].sched; > - kthread_park(sched->thread); > - drm_sched_hw_job_reset(sched, (sched_job->sched == sched ? > + drm_sched_stop(sched, (sched_job->sched == sched ? > sched_job : NULL)); > + > + if(sched_job) > + drm_sched_increase_karma(sched_job); > } > /* get the GPU back into the init state */ > v3d_reset(v3d); > + for (q = 0; q < V3D_MAX_QUEUES; q++) > + drm_sched_resubmit_jobs(sched_job->sched); > + > /* Unblock schedulers and restart their jobs. */ > for (q = 0; q < V3D_MAX_QUEUES; q++) { > - drm_sched_job_recovery(&v3d->queue[q].sched); > - kthread_unpark(v3d->queue[q].sched.thread); > + drm_sched_start(&v3d->queue[q].sched, true); > } > mutex_unlock(&v3d->reset_lock); > diff --git a/include/drm/gpu_scheduler.h > b/include/drm/gpu_scheduler.h > index 47e1979..5ab2d97 100644 > --- a/include/drm/gpu_scheduler.h > +++ b/include/drm/gpu_scheduler.h > @@ -175,6 +175,7 @@ struct drm_sched_fence > *to_drm_sched_fence(struct > dma_fence *f); > * finished to remove the job from the > * @drm_gpu_scheduler.ring_mirror_list. > * @node: used to append this struct to the > @drm_gpu_scheduler.ring_mirror_list. > + * @finish_node: used in a list to wait on before resetting the > scheduler > * @id: a unique id assigned to each job scheduled on the > scheduler. > * @karma: increment on every hang caused by this job. If > this > exceeds the hang > * limit of the scheduler then the job is marked > guilty and > will not > @@ -193,6 +194,7 @@ struct drm_sched_job { > struct dma_fence_cb finish_cb; > struct work_struct finish_work; > struct list_head node; > + struct list_head finish_node; > uint64_t id; > atomic_t karma; > enum drm_sched_priority s_priority; > @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job > *job, > void *owner); > void drm_sched_job_cleanup(struct drm_sched_job *job); > void drm_sched_wakeup(struct drm_gpu_scheduler *sched); > -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, > - struct drm_sched_job *job); > -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); > +void drm_sched_stop(struct drm_gpu_scheduler *sched, > + struct drm_sched_job *job); > +void drm_sched_start(struct drm_gpu_scheduler *sched, bool > full_recovery); > +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); > +void drm_sched_increase_karma(struct drm_sched_job *bad); > bool drm_sched_dependency_optimized(struct dma_fence* fence, > struct drm_sched_entity *entity); > void drm_sched_fault(struct drm_gpu_scheduler *sched); _______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
Am 07.01.19 um 20:47 schrieb Grodzovsky, Andrey:
On 01/07/2019 09:13 AM, Christian König wrote:
Am 03.01.19 um 18:42 schrieb Grodzovsky, Andrey:
On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
On 01/03/2019 03:54 AM, Koenig, Christian wrote:
Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey:
On 12/21/2018 01:37 PM, Christian König wrote: > Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky: >> Decauple sched threads stop and start and ring mirror >> list handling from the policy of what to do about the >> guilty jobs. >> When stoppping the sched thread and detaching sched fences >> from non signaled HW fenes wait for all signaled HW fences >> to complete before rerunning the jobs. >> >> v2: Fix resubmission of guilty job into HW after refactoring. >> >> v4: >> Full restart for all the jobs, not only from guilty ring. >> Extract karma increase into standalone function. >> >> v5: >> Rework waiting for signaled jobs without relying on the job >> struct itself as those might already be freed for non 'guilty' >> job's schedulers. >> Expose karma increase to drivers. >> >> Suggested-by: Christian Koenig Christian.Koenig@amd.com >> Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com >> --- >> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- >> drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- >> drivers/gpu/drm/scheduler/sched_main.c | 188 >> +++++++++++++++++++---------- >> drivers/gpu/drm/v3d/v3d_sched.c | 12 +- >> include/drm/gpu_scheduler.h | 10 +- >> 5 files changed, 151 insertions(+), 88 deletions(-) >> >> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> index 8a078f4..a4bd2d3 100644 >> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >> @@ -3298,12 +3298,10 @@ static int >> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, >> if (!ring || !ring->sched.thread) >> continue; >> - kthread_park(ring->sched.thread); >> + drm_sched_stop(&ring->sched, job ? &job->base : NULL); >> - if (job && job->base.sched != &ring->sched) >> - continue; >> - >> - drm_sched_hw_job_reset(&ring->sched, job ? &job->base : >> NULL); >> + if(job) >> + drm_sched_increase_karma(&job->base); > Since we dropped the "job && job->base.sched != &ring->sched" check > above this will now increase the jobs karma multiple times. > > Maybe just move that outside of the loop. > >> /* after all hw jobs are reset, hw fence is >> meaningless, >> so force_completion */ >> amdgpu_fence_driver_force_completion(ring); >> @@ -3454,14 +3452,10 @@ static void >> amdgpu_device_post_asic_reset(struct amdgpu_device *adev, >> if (!ring || !ring->sched.thread) >> continue; >> - /* only need recovery sched of the given job's ring >> - * or all rings (in the case @job is NULL) >> - * after above amdgpu_reset accomplished >> - */ >> - if ((!job || job->base.sched == &ring->sched) && >> !adev->asic_reset_res) >> - drm_sched_job_recovery(&ring->sched); >> + if (!adev->asic_reset_res) >> + drm_sched_resubmit_jobs(&ring->sched); >> - kthread_unpark(ring->sched.thread); >> + drm_sched_start(&ring->sched, !adev->asic_reset_res); >> } >> if (!amdgpu_device_has_dc_support(adev)) { >> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c >> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c >> index 49a6763..6f1268f 100644 >> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c >> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c >> @@ -109,16 +109,19 @@ static void etnaviv_sched_timedout_job(struct >> drm_sched_job *sched_job) >> } >> /* block scheduler */ >> - kthread_park(gpu->sched.thread); >> - drm_sched_hw_job_reset(&gpu->sched, sched_job); >> + drm_sched_stop(&gpu->sched, sched_job); >> + >> + if(sched_job) >> + drm_sched_increase_karma(sched_job); >> /* get the GPU back into the init state */ >> etnaviv_core_dump(gpu); >> etnaviv_gpu_recover_hang(gpu); >> + drm_sched_resubmit_jobs(&gpu->sched); >> + >> /* restart scheduler after GPU is usable again */ >> - drm_sched_job_recovery(&gpu->sched); >> - kthread_unpark(gpu->sched.thread); >> + drm_sched_start(&gpu->sched, true); >> } >> static void etnaviv_sched_free_job(struct drm_sched_job >> *sched_job) >> diff --git a/drivers/gpu/drm/scheduler/sched_main.c >> b/drivers/gpu/drm/scheduler/sched_main.c >> index dbb6906..b5c5bee 100644 >> --- a/drivers/gpu/drm/scheduler/sched_main.c >> +++ b/drivers/gpu/drm/scheduler/sched_main.c >> @@ -60,8 +60,6 @@ >> static void drm_sched_process_job(struct dma_fence *f, >> struct >> dma_fence_cb *cb); >> -static void drm_sched_expel_job_unlocked(struct >> drm_sched_job >> *s_job); >> - >> /** >> * drm_sched_rq_init - initialize a given run queue struct >> * >> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct >> work_struct *work) >> spin_unlock_irqrestore(&sched->job_list_lock, flags); >> } > Kernel doc here would be nice to have. > >> +void drm_sched_increase_karma(struct drm_sched_job *bad) >> +{ >> + int i; >> + struct drm_sched_entity *tmp; >> + struct drm_sched_entity *entity; >> + struct drm_gpu_scheduler *sched = bad->sched; >> + >> + /* don't increase @bad's karma if it's from KERNEL RQ, >> + * because sometimes GPU hang would cause kernel jobs (like VM >> updating jobs) >> + * corrupt but keep in mind that kernel jobs always considered >> good. >> + */ >> + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { >> + atomic_inc(&bad->karma); >> + for (i = DRM_SCHED_PRIORITY_MIN; i < >> DRM_SCHED_PRIORITY_KERNEL; >> + i++) { >> + struct drm_sched_rq *rq = &sched->sched_rq[i]; >> + >> + spin_lock(&rq->lock); >> + list_for_each_entry_safe(entity, tmp, &rq->entities, >> list) { >> + if (bad->s_fence->scheduled.context == >> + entity->fence_context) { >> + if (atomic_read(&bad->karma) > >> + bad->sched->hang_limit) >> + if (entity->guilty) >> + atomic_set(entity->guilty, 1); >> + break; >> + } >> + } >> + spin_unlock(&rq->lock); >> + if (&entity->list != &rq->entities) >> + break; >> + } >> + } >> +} >> +EXPORT_SYMBOL(drm_sched_increase_karma); >> + >> /** >> * drm_sched_hw_job_reset - stop the scheduler if it >> contains the >> bad job >> * >> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct >> work_struct *work) >> * @bad: bad scheduler job >> * >> */ >> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, >> struct >> drm_sched_job *bad) >> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct >> drm_sched_job *bad) >> { >> - struct drm_sched_job *s_job; >> - struct drm_sched_entity *entity, *tmp; >> + struct drm_sched_job *s_job, *last_job; >> unsigned long flags; >> - int i; >> + struct dma_fence *wait_fence = NULL; >> + int r; >> + >> + kthread_park(sched->thread); >> + /* >> + * Verify all the signaled jobs in mirror list are removed >> from >> the ring >> + * by waiting for their respective scheduler fences to signal. >> + * Continually repeat traversing the ring mirror list >> until no >> more signaled >> + * fences are found >> + */ >> +retry_wait: >> spin_lock_irqsave(&sched->job_list_lock, flags); >> list_for_each_entry_reverse(s_job, >> &sched->ring_mirror_list, >> node) { >> if (s_job->s_fence->parent && >> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct >> drm_gpu_scheduler *sched, struct drm_sched_jo >> dma_fence_put(s_job->s_fence->parent); >> s_job->s_fence->parent = NULL; >> atomic_dec(&sched->hw_rq_count); >> + } else { >> + wait_fence = >> dma_fence_get(&s_job->s_fence->finished); >> + last_job = s_job; >> + break; >> } >> } >> - spin_unlock_irqrestore(&sched->job_list_lock, flags); >> - if (bad && bad->s_priority != >> DRM_SCHED_PRIORITY_KERNEL) { >> - atomic_inc(&bad->karma); >> - /* don't increase @bad's karma if it's from KERNEL RQ, >> - * becuase sometimes GPU hang would cause kernel jobs >> (like >> VM updating jobs) >> - * corrupt but keep in mind that kernel jobs always >> considered good. >> - */ >> - for (i = DRM_SCHED_PRIORITY_MIN; i < >> DRM_SCHED_PRIORITY_KERNEL; i++ ) { >> - struct drm_sched_rq *rq = &sched->sched_rq[i]; >> + /* No signaled jobs in the ring, its safe to proceed to ASIC >> reset */ >> + if (!wait_fence) { >> + spin_unlock_irqrestore(&sched->job_list_lock, flags); >> + goto done; >> + } >> - spin_lock(&rq->lock); >> - list_for_each_entry_safe(entity, tmp, &rq->entities, >> list) { >> - if (bad->s_fence->scheduled.context == >> entity->fence_context) { >> - if (atomic_read(&bad->karma) > >> bad->sched->hang_limit) >> - if (entity->guilty) >> - atomic_set(entity->guilty, 1); >> - break; >> - } >> - } >> - spin_unlock(&rq->lock); >> - if (&entity->list != &rq->entities) >> - break; >> + /* Restore removed cb since removing again already removed >> cb is >> undefined */ >> + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, >> node) { >> + if(s_job == last_job) >> + break; > Need to double check after the holidays, but you should be able > to use > list_for_each_entry_continue here. I think it should work - kind of traversing back on all the jobs we just removed their callbacks.
Wrapping your head around stuff again after the holidays sometimes shows new problems we have completely missed :)
Adding the callbacks again won't work because we have freed up our reference to the hardware fence above:
dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL;
We need to drop this or we would never be able to re-add the fences
Yea, that a big miss on my side...
.
But I'm still not sure if we shouldn't rely on the correct order of signaling and simplify this instead.
As you said before, once we switched to signaling the parent from the interrupt context instead of scheduled work no danger of race there
Correction here - once we switched removing the job from mirror_ring list directly in interrupt context instead later from scheduled work
Ok, so let's stick with the approach of only waiting for the first signaled one found.
But we need to remove setting the parent fence to NULL or otherwise we won't be able to add the callback ever again.
Christian.
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
Andrey
Andrey
, so what if we submit job A and after it job B and B completes before A (like the sync dependency test in libdrm amdgpu tests but without adding explicit dependency to the second command on the first) I believe that still in this case job B's parent (HW) fence will not be signaled before job A completes since EOP event is not signaled until the entire pipe completed and flushed it's cashes including job A. So from this seems to me that indeed it's enough to wait for the last inserted job's parent (HW) fence in ring mirror list to signal. Let me know what you think on that.
P.S V5 is not the last iteration and there was V6 series.
Andrey
Regards, Christian.
Andrey
>> + >> + if (s_job->s_fence->parent) { >> + r = dma_fence_add_callback(s_job->s_fence->parent, >> + &s_job->s_fence->cb, >> + drm_sched_process_job); >> + if (r) >> + DRM_ERROR("fence restore callback failed (%d)\n", >> + r); > When you fail to add the callback this means that you need to call > call drm_sched_process_job manually here. > >> } >> } >> + spin_unlock_irqrestore(&sched->job_list_lock, flags); >> + >> + dma_fence_wait(wait_fence, false); >> + dma_fence_put(wait_fence); >> + wait_fence = NULL; >> + >> + goto retry_wait; >> + >> +done: >> + return; > Drop the done: label and return directly above. > > Apart from all those nit picks that starts to look like it should > work, > Christian. > >> } >> -EXPORT_SYMBOL(drm_sched_hw_job_reset); >> +EXPORT_SYMBOL(drm_sched_stop); >> /** >> * drm_sched_job_recovery - recover jobs after a reset >> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset); >> * @sched: scheduler instance >> * >> */ >> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) >> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool >> full_recovery) >> { >> struct drm_sched_job *s_job, *tmp; >> - bool found_guilty = false; >> unsigned long flags; >> int r; >> + if (!full_recovery) >> + goto unpark; >> + >> spin_lock_irqsave(&sched->job_list_lock, flags); >> list_for_each_entry_safe(s_job, tmp, >> &sched->ring_mirror_list, >> node) { >> struct drm_sched_fence *s_fence = s_job->s_fence; >> - struct dma_fence *fence; >> - uint64_t guilty_context; >> - >> - if (!found_guilty && atomic_read(&s_job->karma) > >> sched->hang_limit) { >> - found_guilty = true; >> - guilty_context = s_job->s_fence->scheduled.context; >> - } >> - >> - if (found_guilty && s_job->s_fence->scheduled.context == >> guilty_context) >> - dma_fence_set_error(&s_fence->finished, -ECANCELED); >> - >> - spin_unlock_irqrestore(&sched->job_list_lock, flags); >> - fence = sched->ops->run_job(s_job); >> - atomic_inc(&sched->hw_rq_count); >> + struct dma_fence *fence = s_job->s_fence->parent; >> if (fence) { >> - s_fence->parent = dma_fence_get(fence); >> r = dma_fence_add_callback(fence, &s_fence->cb, >> drm_sched_process_job); >> if (r == -ENOENT) >> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct >> drm_gpu_scheduler *sched) >> else if (r) >> DRM_ERROR("fence add callback failed (%d)\n", >> r); >> - dma_fence_put(fence); >> - } else { >> - if (s_fence->finished.error < 0) >> - drm_sched_expel_job_unlocked(s_job); >> + } else >> drm_sched_process_job(NULL, &s_fence->cb); >> - } >> - spin_lock_irqsave(&sched->job_list_lock, flags); >> } >> + >> drm_sched_start_timeout(sched); >> spin_unlock_irqrestore(&sched->job_list_lock, flags); >> + >> +unpark: >> + kthread_unpark(sched->thread); >> +} >> +EXPORT_SYMBOL(drm_sched_start); >> + >> +/** >> + * drm_sched_resubmit_jobs - helper to relunch job from mirror >> ring >> list >> + * >> + * @sched: scheduler instance >> + * >> + */ >> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) >> +{ >> + struct drm_sched_job *s_job, *tmp; >> + uint64_t guilty_context; >> + bool found_guilty = false; >> + >> + /*TODO DO we need spinlock here ? */ >> + list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, >> node) { >> + struct drm_sched_fence *s_fence = s_job->s_fence; >> + >> + if (!found_guilty && atomic_read(&s_job->karma) > >> sched->hang_limit) { >> + found_guilty = true; >> + guilty_context = s_job->s_fence->scheduled.context; >> + } >> + >> + if (found_guilty && s_job->s_fence->scheduled.context == >> guilty_context) >> + dma_fence_set_error(&s_fence->finished, -ECANCELED); >> + >> + s_job->s_fence->parent = sched->ops->run_job(s_job); >> + atomic_inc(&sched->hw_rq_count); >> + } >> } >> -EXPORT_SYMBOL(drm_sched_job_recovery); >> +EXPORT_SYMBOL(drm_sched_resubmit_jobs); >> /** >> * drm_sched_job_init - init a scheduler job >> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param) >> DRM_ERROR("fence add callback failed (%d)\n", >> r); >> dma_fence_put(fence); >> - } else { >> - if (s_fence->finished.error < 0) >> - drm_sched_expel_job_unlocked(sched_job); >> + } else >> drm_sched_process_job(NULL, &s_fence->cb); >> - } >> wake_up(&sched->job_scheduled); >> } >> return 0; >> } >> -static void drm_sched_expel_job_unlocked(struct >> drm_sched_job *s_job) >> -{ >> - struct drm_gpu_scheduler *sched = s_job->sched; >> - >> - spin_lock(&sched->job_list_lock); >> - list_del_init(&s_job->node); >> - spin_unlock(&sched->job_list_lock); >> -} >> - >> /** >> * drm_sched_init - Init a gpu scheduler instance >> * >> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c >> b/drivers/gpu/drm/v3d/v3d_sched.c >> index 445b2ef..f76d9ed 100644 >> --- a/drivers/gpu/drm/v3d/v3d_sched.c >> +++ b/drivers/gpu/drm/v3d/v3d_sched.c >> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job >> *sched_job) >> for (q = 0; q < V3D_MAX_QUEUES; q++) { >> struct drm_gpu_scheduler *sched = >> &v3d->queue[q].sched; >> - kthread_park(sched->thread); >> - drm_sched_hw_job_reset(sched, (sched_job->sched == sched ? >> + drm_sched_stop(sched, (sched_job->sched == sched ? >> sched_job : NULL)); >> + >> + if(sched_job) >> + drm_sched_increase_karma(sched_job); >> } >> /* get the GPU back into the init state */ >> v3d_reset(v3d); >> + for (q = 0; q < V3D_MAX_QUEUES; q++) >> + drm_sched_resubmit_jobs(sched_job->sched); >> + >> /* Unblock schedulers and restart their jobs. */ >> for (q = 0; q < V3D_MAX_QUEUES; q++) { >> - drm_sched_job_recovery(&v3d->queue[q].sched); >> - kthread_unpark(v3d->queue[q].sched.thread); >> + drm_sched_start(&v3d->queue[q].sched, true); >> } >> mutex_unlock(&v3d->reset_lock); >> diff --git a/include/drm/gpu_scheduler.h >> b/include/drm/gpu_scheduler.h >> index 47e1979..5ab2d97 100644 >> --- a/include/drm/gpu_scheduler.h >> +++ b/include/drm/gpu_scheduler.h >> @@ -175,6 +175,7 @@ struct drm_sched_fence >> *to_drm_sched_fence(struct >> dma_fence *f); >> * finished to remove the job from the >> * @drm_gpu_scheduler.ring_mirror_list. >> * @node: used to append this struct to the >> @drm_gpu_scheduler.ring_mirror_list. >> + * @finish_node: used in a list to wait on before resetting the >> scheduler >> * @id: a unique id assigned to each job scheduled on the >> scheduler. >> * @karma: increment on every hang caused by this job. If >> this >> exceeds the hang >> * limit of the scheduler then the job is marked >> guilty and >> will not >> @@ -193,6 +194,7 @@ struct drm_sched_job { >> struct dma_fence_cb finish_cb; >> struct work_struct finish_work; >> struct list_head node; >> + struct list_head finish_node; >> uint64_t id; >> atomic_t karma; >> enum drm_sched_priority s_priority; >> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job >> *job, >> void *owner); >> void drm_sched_job_cleanup(struct drm_sched_job *job); >> void drm_sched_wakeup(struct drm_gpu_scheduler *sched); >> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, >> - struct drm_sched_job *job); >> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); >> +void drm_sched_stop(struct drm_gpu_scheduler *sched, >> + struct drm_sched_job *job); >> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool >> full_recovery); >> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); >> +void drm_sched_increase_karma(struct drm_sched_job *bad); >> bool drm_sched_dependency_optimized(struct dma_fence* fence, >> struct drm_sched_entity *entity); >> void drm_sched_fault(struct drm_gpu_scheduler *sched); > _______________________________________________ > amd-gfx mailing list > amd-gfx@lists.freedesktop.org > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/09/2019 05:22 AM, Christian König wrote:
Am 07.01.19 um 20:47 schrieb Grodzovsky, Andrey:
On 01/07/2019 09:13 AM, Christian König wrote:
Am 03.01.19 um 18:42 schrieb Grodzovsky, Andrey:
On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
On 01/03/2019 03:54 AM, Koenig, Christian wrote:
Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey: > On 12/21/2018 01:37 PM, Christian König wrote: >> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky: >>> Decauple sched threads stop and start and ring mirror >>> list handling from the policy of what to do about the >>> guilty jobs. >>> When stoppping the sched thread and detaching sched fences >>> from non signaled HW fenes wait for all signaled HW fences >>> to complete before rerunning the jobs. >>> >>> v2: Fix resubmission of guilty job into HW after refactoring. >>> >>> v4: >>> Full restart for all the jobs, not only from guilty ring. >>> Extract karma increase into standalone function. >>> >>> v5: >>> Rework waiting for signaled jobs without relying on the job >>> struct itself as those might already be freed for non 'guilty' >>> job's schedulers. >>> Expose karma increase to drivers. >>> >>> Suggested-by: Christian Koenig Christian.Koenig@amd.com >>> Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com >>> --- >>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- >>> drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- >>> drivers/gpu/drm/scheduler/sched_main.c | 188 >>> +++++++++++++++++++---------- >>> drivers/gpu/drm/v3d/v3d_sched.c | 12 +- >>> include/drm/gpu_scheduler.h | 10 +- >>> 5 files changed, 151 insertions(+), 88 deletions(-) >>> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>> index 8a078f4..a4bd2d3 100644 >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>> @@ -3298,12 +3298,10 @@ static int >>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, >>> if (!ring || !ring->sched.thread) >>> continue; >>> - kthread_park(ring->sched.thread); >>> + drm_sched_stop(&ring->sched, job ? &job->base : NULL); >>> - if (job && job->base.sched != &ring->sched) >>> - continue; >>> - >>> - drm_sched_hw_job_reset(&ring->sched, job ? &job->base : >>> NULL); >>> + if(job) >>> + drm_sched_increase_karma(&job->base); >> Since we dropped the "job && job->base.sched != &ring->sched" >> check >> above this will now increase the jobs karma multiple times. >> >> Maybe just move that outside of the loop. >> >>> /* after all hw jobs are reset, hw fence is >>> meaningless, >>> so force_completion */ >>> amdgpu_fence_driver_force_completion(ring); >>> @@ -3454,14 +3452,10 @@ static void >>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev, >>> if (!ring || !ring->sched.thread) >>> continue; >>> - /* only need recovery sched of the given job's >>> ring >>> - * or all rings (in the case @job is NULL) >>> - * after above amdgpu_reset accomplished >>> - */ >>> - if ((!job || job->base.sched == &ring->sched) && >>> !adev->asic_reset_res) >>> - drm_sched_job_recovery(&ring->sched); >>> + if (!adev->asic_reset_res) >>> + drm_sched_resubmit_jobs(&ring->sched); >>> - kthread_unpark(ring->sched.thread); >>> + drm_sched_start(&ring->sched, !adev->asic_reset_res); >>> } >>> if (!amdgpu_device_has_dc_support(adev)) { >>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c >>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c >>> index 49a6763..6f1268f 100644 >>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c >>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c >>> @@ -109,16 +109,19 @@ static void >>> etnaviv_sched_timedout_job(struct >>> drm_sched_job *sched_job) >>> } >>> /* block scheduler */ >>> - kthread_park(gpu->sched.thread); >>> - drm_sched_hw_job_reset(&gpu->sched, sched_job); >>> + drm_sched_stop(&gpu->sched, sched_job); >>> + >>> + if(sched_job) >>> + drm_sched_increase_karma(sched_job); >>> /* get the GPU back into the init state */ >>> etnaviv_core_dump(gpu); >>> etnaviv_gpu_recover_hang(gpu); >>> + drm_sched_resubmit_jobs(&gpu->sched); >>> + >>> /* restart scheduler after GPU is usable again */ >>> - drm_sched_job_recovery(&gpu->sched); >>> - kthread_unpark(gpu->sched.thread); >>> + drm_sched_start(&gpu->sched, true); >>> } >>> static void etnaviv_sched_free_job(struct drm_sched_job >>> *sched_job) >>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c >>> b/drivers/gpu/drm/scheduler/sched_main.c >>> index dbb6906..b5c5bee 100644 >>> --- a/drivers/gpu/drm/scheduler/sched_main.c >>> +++ b/drivers/gpu/drm/scheduler/sched_main.c >>> @@ -60,8 +60,6 @@ >>> static void drm_sched_process_job(struct dma_fence *f, >>> struct >>> dma_fence_cb *cb); >>> -static void drm_sched_expel_job_unlocked(struct >>> drm_sched_job >>> *s_job); >>> - >>> /** >>> * drm_sched_rq_init - initialize a given run queue struct >>> * >>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct >>> work_struct *work) >>> spin_unlock_irqrestore(&sched->job_list_lock, flags); >>> } >> Kernel doc here would be nice to have. >> >>> +void drm_sched_increase_karma(struct drm_sched_job *bad) >>> +{ >>> + int i; >>> + struct drm_sched_entity *tmp; >>> + struct drm_sched_entity *entity; >>> + struct drm_gpu_scheduler *sched = bad->sched; >>> + >>> + /* don't increase @bad's karma if it's from KERNEL RQ, >>> + * because sometimes GPU hang would cause kernel jobs >>> (like VM >>> updating jobs) >>> + * corrupt but keep in mind that kernel jobs always >>> considered >>> good. >>> + */ >>> + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { >>> + atomic_inc(&bad->karma); >>> + for (i = DRM_SCHED_PRIORITY_MIN; i < >>> DRM_SCHED_PRIORITY_KERNEL; >>> + i++) { >>> + struct drm_sched_rq *rq = &sched->sched_rq[i]; >>> + >>> + spin_lock(&rq->lock); >>> + list_for_each_entry_safe(entity, tmp, &rq->entities, >>> list) { >>> + if (bad->s_fence->scheduled.context == >>> + entity->fence_context) { >>> + if (atomic_read(&bad->karma) > >>> + bad->sched->hang_limit) >>> + if (entity->guilty) >>> + atomic_set(entity->guilty, 1); >>> + break; >>> + } >>> + } >>> + spin_unlock(&rq->lock); >>> + if (&entity->list != &rq->entities) >>> + break; >>> + } >>> + } >>> +} >>> +EXPORT_SYMBOL(drm_sched_increase_karma); >>> + >>> /** >>> * drm_sched_hw_job_reset - stop the scheduler if it >>> contains the >>> bad job >>> * >>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct >>> work_struct *work) >>> * @bad: bad scheduler job >>> * >>> */ >>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, >>> struct >>> drm_sched_job *bad) >>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct >>> drm_sched_job *bad) >>> { >>> - struct drm_sched_job *s_job; >>> - struct drm_sched_entity *entity, *tmp; >>> + struct drm_sched_job *s_job, *last_job; >>> unsigned long flags; >>> - int i; >>> + struct dma_fence *wait_fence = NULL; >>> + int r; >>> + >>> + kthread_park(sched->thread); >>> + /* >>> + * Verify all the signaled jobs in mirror list are removed >>> from >>> the ring >>> + * by waiting for their respective scheduler fences to >>> signal. >>> + * Continually repeat traversing the ring mirror list >>> until no >>> more signaled >>> + * fences are found >>> + */ >>> +retry_wait: >>> spin_lock_irqsave(&sched->job_list_lock, flags); >>> list_for_each_entry_reverse(s_job, >>> &sched->ring_mirror_list, >>> node) { >>> if (s_job->s_fence->parent && >>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct >>> drm_gpu_scheduler *sched, struct drm_sched_jo >>> dma_fence_put(s_job->s_fence->parent); >>> s_job->s_fence->parent = NULL; >>> atomic_dec(&sched->hw_rq_count); >>> + } else { >>> + wait_fence = >>> dma_fence_get(&s_job->s_fence->finished); >>> + last_job = s_job; >>> + break; >>> } >>> } >>> - spin_unlock_irqrestore(&sched->job_list_lock, flags); >>> - if (bad && bad->s_priority != >>> DRM_SCHED_PRIORITY_KERNEL) { >>> - atomic_inc(&bad->karma); >>> - /* don't increase @bad's karma if it's from KERNEL RQ, >>> - * becuase sometimes GPU hang would cause kernel jobs >>> (like >>> VM updating jobs) >>> - * corrupt but keep in mind that kernel jobs always >>> considered good. >>> - */ >>> - for (i = DRM_SCHED_PRIORITY_MIN; i < >>> DRM_SCHED_PRIORITY_KERNEL; i++ ) { >>> - struct drm_sched_rq *rq = &sched->sched_rq[i]; >>> + /* No signaled jobs in the ring, its safe to proceed to ASIC >>> reset */ >>> + if (!wait_fence) { >>> + spin_unlock_irqrestore(&sched->job_list_lock, flags); >>> + goto done; >>> + } >>> - spin_lock(&rq->lock); >>> - list_for_each_entry_safe(entity, tmp, &rq->entities, >>> list) { >>> - if (bad->s_fence->scheduled.context == >>> entity->fence_context) { >>> - if (atomic_read(&bad->karma) > >>> bad->sched->hang_limit) >>> - if (entity->guilty) >>> - atomic_set(entity->guilty, 1); >>> - break; >>> - } >>> - } >>> - spin_unlock(&rq->lock); >>> - if (&entity->list != &rq->entities) >>> - break; >>> + /* Restore removed cb since removing again already removed >>> cb is >>> undefined */ >>> + list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, >>> node) { >>> + if(s_job == last_job) >>> + break; >> Need to double check after the holidays, but you should be able >> to use >> list_for_each_entry_continue here. > I think it should work - kind of traversing back on all the jobs > we just > removed their callbacks. Wrapping your head around stuff again after the holidays sometimes shows new problems we have completely missed :)
Adding the callbacks again won't work because we have freed up our reference to the hardware fence above: > dma_fence_put(s_job->s_fence->parent); > s_job->s_fence->parent = NULL; We need to drop this or we would never be able to re-add the fences
Yea, that a big miss on my side...
.
But I'm still not sure if we shouldn't rely on the correct order of signaling and simplify this instead.
As you said before, once we switched to signaling the parent from the interrupt context instead of scheduled work no danger of race there
Correction here - once we switched removing the job from mirror_ring list directly in interrupt context instead later from scheduled work
Ok, so let's stick with the approach of only waiting for the first signaled one found.
But we need to remove setting the parent fence to NULL or otherwise we won't be able to add the callback ever again.
Christian.
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Andrey
Andrey
Andrey
, so what if we submit job A and after it job B and B completes before A (like the sync dependency test in libdrm amdgpu tests but without adding explicit dependency to the second command on the first) I believe that still in this case job B's parent (HW) fence will not be signaled before job A completes since EOP event is not signaled until the entire pipe completed and flushed it's cashes including job A. So from this seems to me that indeed it's enough to wait for the last inserted job's parent (HW) fence in ring mirror list to signal. Let me know what you think on that.
P.S V5 is not the last iteration and there was V6 series.
Andrey
Regards, Christian.
> Andrey > >>> + >>> + if (s_job->s_fence->parent) { >>> + r = dma_fence_add_callback(s_job->s_fence->parent, >>> + &s_job->s_fence->cb, >>> + drm_sched_process_job); >>> + if (r) >>> + DRM_ERROR("fence restore callback failed >>> (%d)\n", >>> + r); >> When you fail to add the callback this means that you need to call >> call drm_sched_process_job manually here. >> >>> } >>> } >>> + spin_unlock_irqrestore(&sched->job_list_lock, flags); >>> + >>> + dma_fence_wait(wait_fence, false); >>> + dma_fence_put(wait_fence); >>> + wait_fence = NULL; >>> + >>> + goto retry_wait; >>> + >>> +done: >>> + return; >> Drop the done: label and return directly above. >> >> Apart from all those nit picks that starts to look like it should >> work, >> Christian. >> >>> } >>> -EXPORT_SYMBOL(drm_sched_hw_job_reset); >>> +EXPORT_SYMBOL(drm_sched_stop); >>> /** >>> * drm_sched_job_recovery - recover jobs after a reset >>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset); >>> * @sched: scheduler instance >>> * >>> */ >>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) >>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool >>> full_recovery) >>> { >>> struct drm_sched_job *s_job, *tmp; >>> - bool found_guilty = false; >>> unsigned long flags; >>> int r; >>> + if (!full_recovery) >>> + goto unpark; >>> + >>> spin_lock_irqsave(&sched->job_list_lock, flags); >>> list_for_each_entry_safe(s_job, tmp, >>> &sched->ring_mirror_list, >>> node) { >>> struct drm_sched_fence *s_fence = s_job->s_fence; >>> - struct dma_fence *fence; >>> - uint64_t guilty_context; >>> - >>> - if (!found_guilty && atomic_read(&s_job->karma) > >>> sched->hang_limit) { >>> - found_guilty = true; >>> - guilty_context = s_job->s_fence->scheduled.context; >>> - } >>> - >>> - if (found_guilty && s_job->s_fence->scheduled.context == >>> guilty_context) >>> - dma_fence_set_error(&s_fence->finished, -ECANCELED); >>> - >>> - spin_unlock_irqrestore(&sched->job_list_lock, flags); >>> - fence = sched->ops->run_job(s_job); >>> - atomic_inc(&sched->hw_rq_count); >>> + struct dma_fence *fence = s_job->s_fence->parent; >>> if (fence) { >>> - s_fence->parent = dma_fence_get(fence); >>> r = dma_fence_add_callback(fence, >>> &s_fence->cb, >>> drm_sched_process_job); >>> if (r == -ENOENT) >>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct >>> drm_gpu_scheduler *sched) >>> else if (r) >>> DRM_ERROR("fence add callback failed >>> (%d)\n", >>> r); >>> - dma_fence_put(fence); >>> - } else { >>> - if (s_fence->finished.error < 0) >>> - drm_sched_expel_job_unlocked(s_job); >>> + } else >>> drm_sched_process_job(NULL, &s_fence->cb); >>> - } >>> - spin_lock_irqsave(&sched->job_list_lock, flags); >>> } >>> + >>> drm_sched_start_timeout(sched); >>> spin_unlock_irqrestore(&sched->job_list_lock, flags); >>> + >>> +unpark: >>> + kthread_unpark(sched->thread); >>> +} >>> +EXPORT_SYMBOL(drm_sched_start); >>> + >>> +/** >>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror >>> ring >>> list >>> + * >>> + * @sched: scheduler instance >>> + * >>> + */ >>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) >>> +{ >>> + struct drm_sched_job *s_job, *tmp; >>> + uint64_t guilty_context; >>> + bool found_guilty = false; >>> + >>> + /*TODO DO we need spinlock here ? */ >>> + list_for_each_entry_safe(s_job, tmp, >>> &sched->ring_mirror_list, >>> node) { >>> + struct drm_sched_fence *s_fence = s_job->s_fence; >>> + >>> + if (!found_guilty && atomic_read(&s_job->karma) > >>> sched->hang_limit) { >>> + found_guilty = true; >>> + guilty_context = s_job->s_fence->scheduled.context; >>> + } >>> + >>> + if (found_guilty && s_job->s_fence->scheduled.context == >>> guilty_context) >>> + dma_fence_set_error(&s_fence->finished, -ECANCELED); >>> + >>> + s_job->s_fence->parent = sched->ops->run_job(s_job); >>> + atomic_inc(&sched->hw_rq_count); >>> + } >>> } >>> -EXPORT_SYMBOL(drm_sched_job_recovery); >>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs); >>> /** >>> * drm_sched_job_init - init a scheduler job >>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param) >>> DRM_ERROR("fence add callback failed >>> (%d)\n", >>> r); >>> dma_fence_put(fence); >>> - } else { >>> - if (s_fence->finished.error < 0) >>> - drm_sched_expel_job_unlocked(sched_job); >>> + } else >>> drm_sched_process_job(NULL, &s_fence->cb); >>> - } >>> wake_up(&sched->job_scheduled); >>> } >>> return 0; >>> } >>> -static void drm_sched_expel_job_unlocked(struct >>> drm_sched_job *s_job) >>> -{ >>> - struct drm_gpu_scheduler *sched = s_job->sched; >>> - >>> - spin_lock(&sched->job_list_lock); >>> - list_del_init(&s_job->node); >>> - spin_unlock(&sched->job_list_lock); >>> -} >>> - >>> /** >>> * drm_sched_init - Init a gpu scheduler instance >>> * >>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c >>> b/drivers/gpu/drm/v3d/v3d_sched.c >>> index 445b2ef..f76d9ed 100644 >>> --- a/drivers/gpu/drm/v3d/v3d_sched.c >>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c >>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job >>> *sched_job) >>> for (q = 0; q < V3D_MAX_QUEUES; q++) { >>> struct drm_gpu_scheduler *sched = >>> &v3d->queue[q].sched; >>> - kthread_park(sched->thread); >>> - drm_sched_hw_job_reset(sched, (sched_job->sched == >>> sched ? >>> + drm_sched_stop(sched, (sched_job->sched == sched ? >>> sched_job : NULL)); >>> + >>> + if(sched_job) >>> + drm_sched_increase_karma(sched_job); >>> } >>> /* get the GPU back into the init state */ >>> v3d_reset(v3d); >>> + for (q = 0; q < V3D_MAX_QUEUES; q++) >>> + drm_sched_resubmit_jobs(sched_job->sched); >>> + >>> /* Unblock schedulers and restart their jobs. */ >>> for (q = 0; q < V3D_MAX_QUEUES; q++) { >>> - drm_sched_job_recovery(&v3d->queue[q].sched); >>> - kthread_unpark(v3d->queue[q].sched.thread); >>> + drm_sched_start(&v3d->queue[q].sched, true); >>> } >>> mutex_unlock(&v3d->reset_lock); >>> diff --git a/include/drm/gpu_scheduler.h >>> b/include/drm/gpu_scheduler.h >>> index 47e1979..5ab2d97 100644 >>> --- a/include/drm/gpu_scheduler.h >>> +++ b/include/drm/gpu_scheduler.h >>> @@ -175,6 +175,7 @@ struct drm_sched_fence >>> *to_drm_sched_fence(struct >>> dma_fence *f); >>> * finished to remove the job from the >>> * @drm_gpu_scheduler.ring_mirror_list. >>> * @node: used to append this struct to the >>> @drm_gpu_scheduler.ring_mirror_list. >>> + * @finish_node: used in a list to wait on before resetting the >>> scheduler >>> * @id: a unique id assigned to each job scheduled on the >>> scheduler. >>> * @karma: increment on every hang caused by this job. If >>> this >>> exceeds the hang >>> * limit of the scheduler then the job is marked >>> guilty and >>> will not >>> @@ -193,6 +194,7 @@ struct drm_sched_job { >>> struct dma_fence_cb finish_cb; >>> struct work_struct finish_work; >>> struct list_head node; >>> + struct list_head finish_node; >>> uint64_t id; >>> atomic_t karma; >>> enum drm_sched_priority s_priority; >>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job >>> *job, >>> void *owner); >>> void drm_sched_job_cleanup(struct drm_sched_job *job); >>> void drm_sched_wakeup(struct drm_gpu_scheduler *sched); >>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, >>> - struct drm_sched_job *job); >>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); >>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, >>> + struct drm_sched_job *job); >>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool >>> full_recovery); >>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); >>> +void drm_sched_increase_karma(struct drm_sched_job *bad); >>> bool drm_sched_dependency_optimized(struct dma_fence* >>> fence, >>> struct drm_sched_entity *entity); >>> void drm_sched_fault(struct drm_gpu_scheduler *sched); >> _______________________________________________ >> amd-gfx mailing list >> amd-gfx@lists.freedesktop.org >> https://lists.freedesktop.org/mailman/listinfo/amd-gfx _______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Just a ping.
Andrey
On 01/09/2019 10:18 AM, Andrey Grodzovsky wrote:
On 01/09/2019 05:22 AM, Christian König wrote:
Am 07.01.19 um 20:47 schrieb Grodzovsky, Andrey:
On 01/07/2019 09:13 AM, Christian König wrote:
Am 03.01.19 um 18:42 schrieb Grodzovsky, Andrey:
On 01/03/2019 11:20 AM, Grodzovsky, Andrey wrote:
On 01/03/2019 03:54 AM, Koenig, Christian wrote: > Am 21.12.18 um 21:36 schrieb Grodzovsky, Andrey: >> On 12/21/2018 01:37 PM, Christian König wrote: >>> Am 20.12.18 um 20:23 schrieb Andrey Grodzovsky: >>>> Decauple sched threads stop and start and ring mirror >>>> list handling from the policy of what to do about the >>>> guilty jobs. >>>> When stoppping the sched thread and detaching sched fences >>>> from non signaled HW fenes wait for all signaled HW fences >>>> to complete before rerunning the jobs. >>>> >>>> v2: Fix resubmission of guilty job into HW after refactoring. >>>> >>>> v4: >>>> Full restart for all the jobs, not only from guilty ring. >>>> Extract karma increase into standalone function. >>>> >>>> v5: >>>> Rework waiting for signaled jobs without relying on the job >>>> struct itself as those might already be freed for non 'guilty' >>>> job's schedulers. >>>> Expose karma increase to drivers. >>>> >>>> Suggested-by: Christian Koenig Christian.Koenig@amd.com >>>> Signed-off-by: Andrey Grodzovsky andrey.grodzovsky@amd.com >>>> --- >>>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 18 +-- >>>> drivers/gpu/drm/etnaviv/etnaviv_sched.c | 11 +- >>>> drivers/gpu/drm/scheduler/sched_main.c | 188 >>>> +++++++++++++++++++---------- >>>> drivers/gpu/drm/v3d/v3d_sched.c | 12 +- >>>> include/drm/gpu_scheduler.h | 10 +- >>>> 5 files changed, 151 insertions(+), 88 deletions(-) >>>> >>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>>> index 8a078f4..a4bd2d3 100644 >>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c >>>> @@ -3298,12 +3298,10 @@ static int >>>> amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, >>>> if (!ring || !ring->sched.thread) >>>> continue; >>>> - kthread_park(ring->sched.thread); >>>> + drm_sched_stop(&ring->sched, job ? &job->base : NULL); >>>> - if (job && job->base.sched != &ring->sched) >>>> - continue; >>>> - >>>> - drm_sched_hw_job_reset(&ring->sched, job ? &job->base : >>>> NULL); >>>> + if(job) >>>> + drm_sched_increase_karma(&job->base); >>> Since we dropped the "job && job->base.sched != &ring->sched" >>> check >>> above this will now increase the jobs karma multiple times. >>> >>> Maybe just move that outside of the loop. >>> >>>> /* after all hw jobs are reset, hw fence is >>>> meaningless, >>>> so force_completion */ >>>> amdgpu_fence_driver_force_completion(ring); >>>> @@ -3454,14 +3452,10 @@ static void >>>> amdgpu_device_post_asic_reset(struct amdgpu_device *adev, >>>> if (!ring || !ring->sched.thread) >>>> continue; >>>> - /* only need recovery sched of the given >>>> job's ring >>>> - * or all rings (in the case @job is NULL) >>>> - * after above amdgpu_reset accomplished >>>> - */ >>>> - if ((!job || job->base.sched == &ring->sched) && >>>> !adev->asic_reset_res) >>>> - drm_sched_job_recovery(&ring->sched); >>>> + if (!adev->asic_reset_res) >>>> + drm_sched_resubmit_jobs(&ring->sched); >>>> - kthread_unpark(ring->sched.thread); >>>> + drm_sched_start(&ring->sched, !adev->asic_reset_res); >>>> } >>>> if (!amdgpu_device_has_dc_support(adev)) { >>>> diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c >>>> b/drivers/gpu/drm/etnaviv/etnaviv_sched.c >>>> index 49a6763..6f1268f 100644 >>>> --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c >>>> +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c >>>> @@ -109,16 +109,19 @@ static void >>>> etnaviv_sched_timedout_job(struct >>>> drm_sched_job *sched_job) >>>> } >>>> /* block scheduler */ >>>> - kthread_park(gpu->sched.thread); >>>> - drm_sched_hw_job_reset(&gpu->sched, sched_job); >>>> + drm_sched_stop(&gpu->sched, sched_job); >>>> + >>>> + if(sched_job) >>>> + drm_sched_increase_karma(sched_job); >>>> /* get the GPU back into the init state */ >>>> etnaviv_core_dump(gpu); >>>> etnaviv_gpu_recover_hang(gpu); >>>> + drm_sched_resubmit_jobs(&gpu->sched); >>>> + >>>> /* restart scheduler after GPU is usable again */ >>>> - drm_sched_job_recovery(&gpu->sched); >>>> - kthread_unpark(gpu->sched.thread); >>>> + drm_sched_start(&gpu->sched, true); >>>> } >>>> static void etnaviv_sched_free_job(struct drm_sched_job >>>> *sched_job) >>>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c >>>> b/drivers/gpu/drm/scheduler/sched_main.c >>>> index dbb6906..b5c5bee 100644 >>>> --- a/drivers/gpu/drm/scheduler/sched_main.c >>>> +++ b/drivers/gpu/drm/scheduler/sched_main.c >>>> @@ -60,8 +60,6 @@ >>>> static void drm_sched_process_job(struct dma_fence *f, >>>> struct >>>> dma_fence_cb *cb); >>>> -static void drm_sched_expel_job_unlocked(struct >>>> drm_sched_job >>>> *s_job); >>>> - >>>> /** >>>> * drm_sched_rq_init - initialize a given run queue >>>> struct >>>> * >>>> @@ -335,6 +333,42 @@ static void drm_sched_job_timedout(struct >>>> work_struct *work) >>>> spin_unlock_irqrestore(&sched->job_list_lock, flags); >>>> } >>> Kernel doc here would be nice to have. >>> >>>> +void drm_sched_increase_karma(struct drm_sched_job *bad) >>>> +{ >>>> + int i; >>>> + struct drm_sched_entity *tmp; >>>> + struct drm_sched_entity *entity; >>>> + struct drm_gpu_scheduler *sched = bad->sched; >>>> + >>>> + /* don't increase @bad's karma if it's from KERNEL RQ, >>>> + * because sometimes GPU hang would cause kernel jobs >>>> (like VM >>>> updating jobs) >>>> + * corrupt but keep in mind that kernel jobs always >>>> considered >>>> good. >>>> + */ >>>> + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { >>>> + atomic_inc(&bad->karma); >>>> + for (i = DRM_SCHED_PRIORITY_MIN; i < >>>> DRM_SCHED_PRIORITY_KERNEL; >>>> + i++) { >>>> + struct drm_sched_rq *rq = &sched->sched_rq[i]; >>>> + >>>> + spin_lock(&rq->lock); >>>> + list_for_each_entry_safe(entity, tmp, >>>> &rq->entities, >>>> list) { >>>> + if (bad->s_fence->scheduled.context == >>>> + entity->fence_context) { >>>> + if (atomic_read(&bad->karma) > >>>> + bad->sched->hang_limit) >>>> + if (entity->guilty) >>>> + atomic_set(entity->guilty, 1); >>>> + break; >>>> + } >>>> + } >>>> + spin_unlock(&rq->lock); >>>> + if (&entity->list != &rq->entities) >>>> + break; >>>> + } >>>> + } >>>> +} >>>> +EXPORT_SYMBOL(drm_sched_increase_karma); >>>> + >>>> /** >>>> * drm_sched_hw_job_reset - stop the scheduler if it >>>> contains the >>>> bad job >>>> * >>>> @@ -342,13 +376,22 @@ static void drm_sched_job_timedout(struct >>>> work_struct *work) >>>> * @bad: bad scheduler job >>>> * >>>> */ >>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, >>>> struct >>>> drm_sched_job *bad) >>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, struct >>>> drm_sched_job *bad) >>>> { >>>> - struct drm_sched_job *s_job; >>>> - struct drm_sched_entity *entity, *tmp; >>>> + struct drm_sched_job *s_job, *last_job; >>>> unsigned long flags; >>>> - int i; >>>> + struct dma_fence *wait_fence = NULL; >>>> + int r; >>>> + >>>> + kthread_park(sched->thread); >>>> + /* >>>> + * Verify all the signaled jobs in mirror list are removed >>>> from >>>> the ring >>>> + * by waiting for their respective scheduler fences to >>>> signal. >>>> + * Continually repeat traversing the ring mirror list >>>> until no >>>> more signaled >>>> + * fences are found >>>> + */ >>>> +retry_wait: >>>> spin_lock_irqsave(&sched->job_list_lock, flags); >>>> list_for_each_entry_reverse(s_job, >>>> &sched->ring_mirror_list, >>>> node) { >>>> if (s_job->s_fence->parent && >>>> @@ -357,35 +400,45 @@ void drm_sched_hw_job_reset(struct >>>> drm_gpu_scheduler *sched, struct drm_sched_jo >>>> dma_fence_put(s_job->s_fence->parent); >>>> s_job->s_fence->parent = NULL; >>>> atomic_dec(&sched->hw_rq_count); >>>> + } else { >>>> + wait_fence = >>>> dma_fence_get(&s_job->s_fence->finished); >>>> + last_job = s_job; >>>> + break; >>>> } >>>> } >>>> - spin_unlock_irqrestore(&sched->job_list_lock, flags); >>>> - if (bad && bad->s_priority != >>>> DRM_SCHED_PRIORITY_KERNEL) { >>>> - atomic_inc(&bad->karma); >>>> - /* don't increase @bad's karma if it's from KERNEL RQ, >>>> - * becuase sometimes GPU hang would cause kernel jobs >>>> (like >>>> VM updating jobs) >>>> - * corrupt but keep in mind that kernel jobs always >>>> considered good. >>>> - */ >>>> - for (i = DRM_SCHED_PRIORITY_MIN; i < >>>> DRM_SCHED_PRIORITY_KERNEL; i++ ) { >>>> - struct drm_sched_rq *rq = &sched->sched_rq[i]; >>>> + /* No signaled jobs in the ring, its safe to proceed to >>>> ASIC >>>> reset */ >>>> + if (!wait_fence) { >>>> + spin_unlock_irqrestore(&sched->job_list_lock, flags); >>>> + goto done; >>>> + } >>>> - spin_lock(&rq->lock); >>>> - list_for_each_entry_safe(entity, tmp, >>>> &rq->entities, >>>> list) { >>>> - if (bad->s_fence->scheduled.context == >>>> entity->fence_context) { >>>> - if (atomic_read(&bad->karma) > >>>> bad->sched->hang_limit) >>>> - if (entity->guilty) >>>> - atomic_set(entity->guilty, 1); >>>> - break; >>>> - } >>>> - } >>>> - spin_unlock(&rq->lock); >>>> - if (&entity->list != &rq->entities) >>>> - break; >>>> + /* Restore removed cb since removing again already removed >>>> cb is >>>> undefined */ >>>> + list_for_each_entry_reverse(s_job, >>>> &sched->ring_mirror_list, >>>> node) { >>>> + if(s_job == last_job) >>>> + break; >>> Need to double check after the holidays, but you should be able >>> to use >>> list_for_each_entry_continue here. >> I think it should work - kind of traversing back on all the jobs >> we just >> removed their callbacks. > Wrapping your head around stuff again after the holidays sometimes > shows > new problems we have completely missed :) > > Adding the callbacks again won't work because we have freed up our > reference to the hardware fence above: >> dma_fence_put(s_job->s_fence->parent); >> s_job->s_fence->parent = NULL; > We need to drop this or we would never be able to re-add the fences Yea, that a big miss on my side...
> . > > But I'm still not sure if we shouldn't rely on the correct order of > signaling and simplify this instead. As you said before, once we switched to signaling the parent from the interrupt context instead of scheduled work no danger of race there
Correction here - once we switched removing the job from mirror_ring list directly in interrupt context instead later from scheduled work
Ok, so let's stick with the approach of only waiting for the first signaled one found.
But we need to remove setting the parent fence to NULL or otherwise we won't be able to add the callback ever again.
Christian.
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Andrey
Andrey
Andrey
, so what if we submit job A and after it job B and B completes before A (like the sync dependency test in libdrm amdgpu tests but without adding explicit dependency to the second command on the first) I believe that still in this case job B's parent (HW) fence will not be signaled before job A completes since EOP event is not signaled until the entire pipe completed and flushed it's cashes including job A. So from this seems to me that indeed it's enough to wait for the last inserted job's parent (HW) fence in ring mirror list to signal. Let me know what you think on that.
P.S V5 is not the last iteration and there was V6 series.
Andrey
> Regards, > Christian. > >> Andrey >> >>>> + >>>> + if (s_job->s_fence->parent) { >>>> + r = dma_fence_add_callback(s_job->s_fence->parent, >>>> + &s_job->s_fence->cb, >>>> + drm_sched_process_job); >>>> + if (r) >>>> + DRM_ERROR("fence restore callback failed >>>> (%d)\n", >>>> + r); >>> When you fail to add the callback this means that you need to >>> call >>> call drm_sched_process_job manually here. >>> >>>> } >>>> } >>>> + spin_unlock_irqrestore(&sched->job_list_lock, flags); >>>> + >>>> + dma_fence_wait(wait_fence, false); >>>> + dma_fence_put(wait_fence); >>>> + wait_fence = NULL; >>>> + >>>> + goto retry_wait; >>>> + >>>> +done: >>>> + return; >>> Drop the done: label and return directly above. >>> >>> Apart from all those nit picks that starts to look like it should >>> work, >>> Christian. >>> >>>> } >>>> -EXPORT_SYMBOL(drm_sched_hw_job_reset); >>>> +EXPORT_SYMBOL(drm_sched_stop); >>>> /** >>>> * drm_sched_job_recovery - recover jobs after a reset >>>> @@ -393,33 +446,21 @@ EXPORT_SYMBOL(drm_sched_hw_job_reset); >>>> * @sched: scheduler instance >>>> * >>>> */ >>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched) >>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool >>>> full_recovery) >>>> { >>>> struct drm_sched_job *s_job, *tmp; >>>> - bool found_guilty = false; >>>> unsigned long flags; >>>> int r; >>>> + if (!full_recovery) >>>> + goto unpark; >>>> + >>>> spin_lock_irqsave(&sched->job_list_lock, flags); >>>> list_for_each_entry_safe(s_job, tmp, >>>> &sched->ring_mirror_list, >>>> node) { >>>> struct drm_sched_fence *s_fence = s_job->s_fence; >>>> - struct dma_fence *fence; >>>> - uint64_t guilty_context; >>>> - >>>> - if (!found_guilty && atomic_read(&s_job->karma) > >>>> sched->hang_limit) { >>>> - found_guilty = true; >>>> - guilty_context = s_job->s_fence->scheduled.context; >>>> - } >>>> - >>>> - if (found_guilty && >>>> s_job->s_fence->scheduled.context == >>>> guilty_context) >>>> - dma_fence_set_error(&s_fence->finished, -ECANCELED); >>>> - >>>> - spin_unlock_irqrestore(&sched->job_list_lock, flags); >>>> - fence = sched->ops->run_job(s_job); >>>> - atomic_inc(&sched->hw_rq_count); >>>> + struct dma_fence *fence = s_job->s_fence->parent; >>>> if (fence) { >>>> - s_fence->parent = dma_fence_get(fence); >>>> r = dma_fence_add_callback(fence, >>>> &s_fence->cb, >>>> drm_sched_process_job); >>>> if (r == -ENOENT) >>>> @@ -427,18 +468,47 @@ void drm_sched_job_recovery(struct >>>> drm_gpu_scheduler *sched) >>>> else if (r) >>>> DRM_ERROR("fence add callback failed >>>> (%d)\n", >>>> r); >>>> - dma_fence_put(fence); >>>> - } else { >>>> - if (s_fence->finished.error < 0) >>>> - drm_sched_expel_job_unlocked(s_job); >>>> + } else >>>> drm_sched_process_job(NULL, &s_fence->cb); >>>> - } >>>> - spin_lock_irqsave(&sched->job_list_lock, flags); >>>> } >>>> + >>>> drm_sched_start_timeout(sched); >>>> spin_unlock_irqrestore(&sched->job_list_lock, flags); >>>> + >>>> +unpark: >>>> + kthread_unpark(sched->thread); >>>> +} >>>> +EXPORT_SYMBOL(drm_sched_start); >>>> + >>>> +/** >>>> + * drm_sched_resubmit_jobs - helper to relunch job from mirror >>>> ring >>>> list >>>> + * >>>> + * @sched: scheduler instance >>>> + * >>>> + */ >>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) >>>> +{ >>>> + struct drm_sched_job *s_job, *tmp; >>>> + uint64_t guilty_context; >>>> + bool found_guilty = false; >>>> + >>>> + /*TODO DO we need spinlock here ? */ >>>> + list_for_each_entry_safe(s_job, tmp, >>>> &sched->ring_mirror_list, >>>> node) { >>>> + struct drm_sched_fence *s_fence = s_job->s_fence; >>>> + >>>> + if (!found_guilty && atomic_read(&s_job->karma) > >>>> sched->hang_limit) { >>>> + found_guilty = true; >>>> + guilty_context = s_job->s_fence->scheduled.context; >>>> + } >>>> + >>>> + if (found_guilty && >>>> s_job->s_fence->scheduled.context == >>>> guilty_context) >>>> + dma_fence_set_error(&s_fence->finished, -ECANCELED); >>>> + >>>> + s_job->s_fence->parent = sched->ops->run_job(s_job); >>>> + atomic_inc(&sched->hw_rq_count); >>>> + } >>>> } >>>> -EXPORT_SYMBOL(drm_sched_job_recovery); >>>> +EXPORT_SYMBOL(drm_sched_resubmit_jobs); >>>> /** >>>> * drm_sched_job_init - init a scheduler job >>>> @@ -634,26 +704,14 @@ static int drm_sched_main(void *param) >>>> DRM_ERROR("fence add callback failed >>>> (%d)\n", >>>> r); >>>> dma_fence_put(fence); >>>> - } else { >>>> - if (s_fence->finished.error < 0) >>>> - drm_sched_expel_job_unlocked(sched_job); >>>> + } else >>>> drm_sched_process_job(NULL, &s_fence->cb); >>>> - } >>>> wake_up(&sched->job_scheduled); >>>> } >>>> return 0; >>>> } >>>> -static void drm_sched_expel_job_unlocked(struct >>>> drm_sched_job *s_job) >>>> -{ >>>> - struct drm_gpu_scheduler *sched = s_job->sched; >>>> - >>>> - spin_lock(&sched->job_list_lock); >>>> - list_del_init(&s_job->node); >>>> - spin_unlock(&sched->job_list_lock); >>>> -} >>>> - >>>> /** >>>> * drm_sched_init - Init a gpu scheduler instance >>>> * >>>> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c >>>> b/drivers/gpu/drm/v3d/v3d_sched.c >>>> index 445b2ef..f76d9ed 100644 >>>> --- a/drivers/gpu/drm/v3d/v3d_sched.c >>>> +++ b/drivers/gpu/drm/v3d/v3d_sched.c >>>> @@ -178,18 +178,22 @@ v3d_job_timedout(struct drm_sched_job >>>> *sched_job) >>>> for (q = 0; q < V3D_MAX_QUEUES; q++) { >>>> struct drm_gpu_scheduler *sched = >>>> &v3d->queue[q].sched; >>>> - kthread_park(sched->thread); >>>> - drm_sched_hw_job_reset(sched, (sched_job->sched == >>>> sched ? >>>> + drm_sched_stop(sched, (sched_job->sched == sched ? >>>> sched_job : NULL)); >>>> + >>>> + if(sched_job) >>>> + drm_sched_increase_karma(sched_job); >>>> } >>>> /* get the GPU back into the init state */ >>>> v3d_reset(v3d); >>>> + for (q = 0; q < V3D_MAX_QUEUES; q++) >>>> + drm_sched_resubmit_jobs(sched_job->sched); >>>> + >>>> /* Unblock schedulers and restart their jobs. */ >>>> for (q = 0; q < V3D_MAX_QUEUES; q++) { >>>> - drm_sched_job_recovery(&v3d->queue[q].sched); >>>> - kthread_unpark(v3d->queue[q].sched.thread); >>>> + drm_sched_start(&v3d->queue[q].sched, true); >>>> } >>>> mutex_unlock(&v3d->reset_lock); >>>> diff --git a/include/drm/gpu_scheduler.h >>>> b/include/drm/gpu_scheduler.h >>>> index 47e1979..5ab2d97 100644 >>>> --- a/include/drm/gpu_scheduler.h >>>> +++ b/include/drm/gpu_scheduler.h >>>> @@ -175,6 +175,7 @@ struct drm_sched_fence >>>> *to_drm_sched_fence(struct >>>> dma_fence *f); >>>> * finished to remove the job from the >>>> * @drm_gpu_scheduler.ring_mirror_list. >>>> * @node: used to append this struct to the >>>> @drm_gpu_scheduler.ring_mirror_list. >>>> + * @finish_node: used in a list to wait on before resetting the >>>> scheduler >>>> * @id: a unique id assigned to each job scheduled on the >>>> scheduler. >>>> * @karma: increment on every hang caused by this job. If >>>> this >>>> exceeds the hang >>>> * limit of the scheduler then the job is marked >>>> guilty and >>>> will not >>>> @@ -193,6 +194,7 @@ struct drm_sched_job { >>>> struct dma_fence_cb finish_cb; >>>> struct work_struct finish_work; >>>> struct list_head node; >>>> + struct list_head finish_node; >>>> uint64_t id; >>>> atomic_t karma; >>>> enum drm_sched_priority s_priority; >>>> @@ -298,9 +300,11 @@ int drm_sched_job_init(struct drm_sched_job >>>> *job, >>>> void *owner); >>>> void drm_sched_job_cleanup(struct drm_sched_job *job); >>>> void drm_sched_wakeup(struct drm_gpu_scheduler *sched); >>>> -void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, >>>> - struct drm_sched_job *job); >>>> -void drm_sched_job_recovery(struct drm_gpu_scheduler *sched); >>>> +void drm_sched_stop(struct drm_gpu_scheduler *sched, >>>> + struct drm_sched_job *job); >>>> +void drm_sched_start(struct drm_gpu_scheduler *sched, bool >>>> full_recovery); >>>> +void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); >>>> +void drm_sched_increase_karma(struct drm_sched_job *bad); >>>> bool drm_sched_dependency_optimized(struct dma_fence* >>>> fence, >>>> struct drm_sched_entity *entity); >>>> void drm_sched_fault(struct drm_gpu_scheduler *sched); >>> _______________________________________________ >>> amd-gfx mailing list >>> amd-gfx@lists.freedesktop.org >>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx > _______________________________________________ > amd-gfx mailing list > amd-gfx@lists.freedesktop.org > https://lists.freedesktop.org/mailman/listinfo/amd-gfx _______________________________________________ dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Andrey
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Andrey
Andrey
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
Andrey
Andrey
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/11/2019 02:11 PM, Koenig, Christian wrote:
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
> But we will not be adding the cb back in drm_sched_stop anymore, now we > are only going to add back the cb in drm_sched_startr after rerunning > those jobs in drm_sched_resubmit_jobs and assign them a new parent > there > anyway. Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
Andrey
Andrey
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
On 01/11/2019 02:11 PM, Koenig, Christian wrote:
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
>> But we will not be adding the cb back in drm_sched_stop >> anymore, now we >> are only going to add back the cb in drm_sched_startr after >> rerunning >> those jobs in drm_sched_resubmit_jobs and assign them a new parent >> there >> anyway. > Yeah, but when we find that we don't need to reset anything anymore > then adding the callbacks again won't be possible any more. > > Christian. I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
I implemented HW reset avoidance including XGMI use case according to the plan i specified. Patch is attached but I can't test it yet due to XGMI regression in PSP which is supposed to be fixed soon. Please take a look.
Andrey
Andrey
Andrey
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
On 01/11/2019 02:11 PM, Koenig, Christian wrote:
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP] >>> But we will not be adding the cb back in drm_sched_stop >>> anymore, now we >>> are only going to add back the cb in drm_sched_startr after >>> rerunning >>> those jobs in drm_sched_resubmit_jobs and assign them a new parent >>> there >>> anyway. >> Yeah, but when we find that we don't need to reset anything anymore >> then adding the callbacks again won't be possible any more. >> >> Christian. > I am not sure I understand it, can u point me to example of how this > will happen ? I am attaching my latest patches with waiting only for > the last job's fence here just so we are on same page regarding > the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
I implemented HW reset avoidance including XGMI use case according to the plan i specified. Patch is attached but I can't test it yet due to XGMI regression in PSP which is supposed to be fixed soon. Please take a look.
Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.
Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.
How about the following handling: 1. Add the timeout job to a list. 2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately. 3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks. 4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything. 5. Do the reset on all affected devices. 6. Drop the lock. 7. Add callbacks again and restart the schedulers.
Regards, Christian.
Andrey
Andrey
> Andrey >
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/16/2019 02:46 AM, Christian König wrote: Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
On 01/11/2019 02:11 PM, Koenig, Christian wrote:
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
I implemented HW reset avoidance including XGMI use case according to the plan i specified. Patch is attached but I can't test it yet due to XGMI regression in PSP which is supposed to be fixed soon. Please take a look.
Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.
Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.
Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.
How about the following handling: 1. Add the timeout job to a list. 2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately. 3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks. 4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.
What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we miss a required reset in this case and wouldn't even have any indication of their hang ?
5. Do the reset on all affected devices. 6. Drop the lock. 7. Add callbacks again and restart the schedulers.
I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out list ?
Andrey
Regards, Christian.
Andrey
Andrey
Andrey
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:
On 01/16/2019 02:46 AM, Christian König wrote: Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
On 01/11/2019 02:11 PM, Koenig, Christian wrote:
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
I implemented HW reset avoidance including XGMI use case according to the plan i specified. Patch is attached but I can't test it yet due to XGMI regression in PSP which is supposed to be fixed soon. Please take a look.
Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.
Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.
Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.
How about the following handling: 1. Add the timeout job to a list. 2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately. 3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks. 4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.
What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we miss a required reset in this case and wouldn't even have any indication of their hang ?
If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.
If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.
Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.
See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.
Christian.
5. Do the reset on all affected devices. 6. Drop the lock. 7. Add callbacks again and restart the schedulers.
I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out list ?
Andrey
Regards, Christian.
Andrey
Andrey
Andrey
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/16/2019 11:02 AM, Koenig, Christian wrote: Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:
On 01/16/2019 02:46 AM, Christian König wrote: Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
On 01/11/2019 02:11 PM, Koenig, Christian wrote:
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
I implemented HW reset avoidance including XGMI use case according to the plan i specified. Patch is attached but I can't test it yet due to XGMI regression in PSP which is supposed to be fixed soon. Please take a look.
Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.
Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.
Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.
How about the following handling: 1. Add the timeout job to a list. 2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately. 3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks. 4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.
What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we miss a required reset in this case and wouldn't even have any indication of their hang ?
If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.
If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.
Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.
See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.
Christian.
This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger - is this OK ? About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI. What do you think ?
Andrey
5. Do the reset on all affected devices. 6. Drop the lock. 7. Add callbacks again and restart the schedulers.
I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out list ?
Andrey
Regards, Christian.
Andrey
Andrey
Andrey
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 16.01.19 um 18:17 schrieb Grodzovsky, Andrey:
On 01/16/2019 11:02 AM, Koenig, Christian wrote:
Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:
On 01/16/2019 02:46 AM, Christian König wrote:
Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
On 01/11/2019 02:11 PM, Koenig, Christian wrote: > Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey: >> On 01/11/2019 04:42 AM, Koenig, Christian wrote: >>> Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey: >>>> [SNIP] >>>>>>> But we will not be adding the cb back in drm_sched_stop >>>>>>> anymore, now we >>>>>>> are only going to add back the cb in drm_sched_startr after >>>>>>> rerunning >>>>>>> those jobs in drm_sched_resubmit_jobs and assign them a new parent >>>>>>> there >>>>>>> anyway. >>>>>> Yeah, but when we find that we don't need to reset anything anymore >>>>>> then adding the callbacks again won't be possible any more. >>>>>> >>>>>> Christian. >>>>> I am not sure I understand it, can u point me to example of how this >>>>> will happen ? I am attaching my latest patches with waiting only for >>>>> the last job's fence here just so we are on same page regarding >>>>> the code. >>> Well the whole idea is to prepare all schedulers, then check once more >>> if the offending job hasn't completed in the meantime. >>> >>> If the job completed we need to be able to rollback everything and >>> continue as if nothing had happened. >>> >>> Christian. >> Oh, but this piece of functionality - skipping HW ASIC reset in case >> the >> guilty job done is totally missing form this patch series and so needs >> to be added. So what you say actually is that for the case were we skip >> HW asic reset because the guilty job did complete we also need to skip >> resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the >> old parent fence pointer for reuse ? If so I would like to add all this >> functionality as a third patch since the first 2 patches are more about >> resolving race condition with jobs in flight while doing reset - >> what do >> you think ? > Yeah, sounds perfectly fine to me. > > Christian. I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
I implemented HW reset avoidance including XGMI use case according to the plan i specified. Patch is attached but I can't test it yet due to XGMI regression in PSP which is supposed to be fixed soon. Please take a look.
Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.
Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.
Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.
How about the following handling:
- Add the timeout job to a list.
- Try to grab a lock to handle the reset, if that doesn't work
because there is already a reset underway return immediately. 3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks. 4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.
What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we miss a required reset in this case and wouldn't even have any indication of their hang ?
If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.
If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.
Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.
See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.
Christian.
This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger
- is this OK ?
Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.
About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI. What do you think ?
Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.
Christian.
Andrey
- Do the reset on all affected devices.
- Drop the lock.
- Add callbacks again and restart the schedulers.
I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out list ?
Andrey
Regards, Christian.
Andrey
>> Andrey >>>>> Andrey >>>>> >>> _______________________________________________ >>> amd-gfx mailing list >>> amd-gfx@lists.freedesktop.org >>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx > _______________________________________________ > amd-gfx mailing list > amd-gfx@lists.freedesktop.org > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/17/2019 02:45 AM, Christian König wrote: Am 16.01.19 um 18:17 schrieb Grodzovsky, Andrey:
On 01/16/2019 11:02 AM, Koenig, Christian wrote: Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:
On 01/16/2019 02:46 AM, Christian König wrote: Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
On 01/11/2019 02:11 PM, Koenig, Christian wrote:
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
I implemented HW reset avoidance including XGMI use case according to the plan i specified. Patch is attached but I can't test it yet due to XGMI regression in PSP which is supposed to be fixed soon. Please take a look.
Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.
Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.
Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.
How about the following handling: 1. Add the timeout job to a list. 2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately. 3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks. 4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.
What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we miss a required reset in this case and wouldn't even have any indication of their hang ?
If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.
If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.
Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.
See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.
Christian.
This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger - is this OK ?
Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.
By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?
About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI. What do you think ?
Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.
Christian.
But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.
I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does imply drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?
Andrey
Andrey
5. Do the reset on all affected devices. 6. Drop the lock. 7. Add callbacks again and restart the schedulers.
I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out list ?
Andrey
Regards, Christian.
Andrey
Andrey
Andrey
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 17.01.19 um 16:22 schrieb Grodzovsky, Andrey:
On 01/17/2019 02:45 AM, Christian König wrote: Am 16.01.19 um 18:17 schrieb Grodzovsky, Andrey:
On 01/16/2019 11:02 AM, Koenig, Christian wrote: Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:
On 01/16/2019 02:46 AM, Christian König wrote: Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
On 01/11/2019 02:11 PM, Koenig, Christian wrote:
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
I implemented HW reset avoidance including XGMI use case according to the plan i specified. Patch is attached but I can't test it yet due to XGMI regression in PSP which is supposed to be fixed soon. Please take a look.
Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.
Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.
Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.
How about the following handling: 1. Add the timeout job to a list. 2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately. 3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks. 4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.
What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we miss a required reset in this case and wouldn't even have any indication of their hang ?
If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.
If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.
Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.
See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.
Christian.
This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger - is this OK ?
Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.
By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?
Correct, yes.
About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI. What do you think ?
Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.
Christian.
But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.
That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.
I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does imply drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?
Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.
What we could do is to make it a timer instead and raise the work item from the device specific callback.
But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.
Christian.
Andrey
Andrey
5. Do the reset on all affected devices. 6. Drop the lock. 7. Add callbacks again and restart the schedulers.
I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out list ?
Andrey
Regards, Christian.
Andrey
Andrey
Andrey
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/17/2019 10:29 AM, Koenig, Christian wrote: Am 17.01.19 um 16:22 schrieb Grodzovsky, Andrey:
On 01/17/2019 02:45 AM, Christian König wrote: Am 16.01.19 um 18:17 schrieb Grodzovsky, Andrey:
On 01/16/2019 11:02 AM, Koenig, Christian wrote: Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:
On 01/16/2019 02:46 AM, Christian König wrote: Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
On 01/11/2019 02:11 PM, Koenig, Christian wrote:
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
I implemented HW reset avoidance including XGMI use case according to the plan i specified. Patch is attached but I can't test it yet due to XGMI regression in PSP which is supposed to be fixed soon. Please take a look.
Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.
Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.
Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.
How about the following handling: 1. Add the timeout job to a list. 2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately. 3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks. 4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.
What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we miss a required reset in this case and wouldn't even have any indication of their hang ?
If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.
If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.
Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.
See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.
Christian.
This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger - is this OK ?
Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.
By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?
Correct, yes.
About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI. What do you think ?
Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.
Christian.
But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.
That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.
Yes, i realize it's not needed any more once we give up on time out list idea.
I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does imply drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?
Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.
What we could do is to make it a timer instead and raise the work item from the device specific callback.
But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.
Christian.
No it's not, it's just saves us all the locking dance we are doing in amdgpu_device_gpu_recover code, especially for XGMI, but it does introduce other complications I guess. OK, i will rework the reset optimization patch as we discussed and resubmit the series again.
Andrey
Andrey
Andrey
5. Do the reset on all affected devices. 6. Drop the lock. 7. Add callbacks again and restart the schedulers.
I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out list ?
Andrey
Regards, Christian.
Andrey
Andrey
Andrey
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/17/2019 10:29 AM, Koenig, Christian wrote: Am 17.01.19 um 16:22 schrieb Grodzovsky, Andrey:
On 01/17/2019 02:45 AM, Christian König wrote: Am 16.01.19 um 18:17 schrieb Grodzovsky, Andrey:
On 01/16/2019 11:02 AM, Koenig, Christian wrote: Am 16.01.19 um 16:45 schrieb Grodzovsky, Andrey:
On 01/16/2019 02:46 AM, Christian König wrote: Am 15.01.19 um 23:01 schrieb Grodzovsky, Andrey:
On 01/11/2019 05:03 PM, Andrey Grodzovsky wrote:
On 01/11/2019 02:11 PM, Koenig, Christian wrote:
Am 11.01.19 um 16:37 schrieb Grodzovsky, Andrey:
On 01/11/2019 04:42 AM, Koenig, Christian wrote:
Am 10.01.19 um 16:56 schrieb Grodzovsky, Andrey:
[SNIP]
But we will not be adding the cb back in drm_sched_stop anymore, now we are only going to add back the cb in drm_sched_startr after rerunning those jobs in drm_sched_resubmit_jobs and assign them a new parent there anyway.
Yeah, but when we find that we don't need to reset anything anymore then adding the callbacks again won't be possible any more.
Christian.
I am not sure I understand it, can u point me to example of how this will happen ? I am attaching my latest patches with waiting only for the last job's fence here just so we are on same page regarding the code.
Well the whole idea is to prepare all schedulers, then check once more if the offending job hasn't completed in the meantime.
If the job completed we need to be able to rollback everything and continue as if nothing had happened.
Christian.
Oh, but this piece of functionality - skipping HW ASIC reset in case the guilty job done is totally missing form this patch series and so needs to be added. So what you say actually is that for the case were we skip HW asic reset because the guilty job did complete we also need to skip resubmitting the jobs in drm_sched_resubmit_jobs and hence preserve the old parent fence pointer for reuse ? If so I would like to add all this functionality as a third patch since the first 2 patches are more about resolving race condition with jobs in flight while doing reset - what do you think ?
Yeah, sounds perfectly fine to me.
Christian.
I realized there is another complication now for XGMI hive use case, we currently skip gpu recover for adev in case another gpu recover for different adev in same hive is running, under the assumption that we are going to reset all devices in hive anyway because that should cover our own dev too. But if we chose to skip now HW asic reset if our guilty job did finish we will aslo not HW reset any other devices in the hive even if one of them might actually had a bad job, wanted to do gpu recover but skipped it because our recover was in progress in that time. My general idea on that is to keep a list of guilty jobs per hive, when you start gpu recover you first add you guilty job to the hive and trylock hive->reset_lock. Any owner of hive->reset_lock (gpu recovery in progress) once he finished his recovery and released hive->reset_lock should go over hive->bad_jobs_list and if at least one of them is still not signaled (not done) trigger another gpu recovery and so on. If you do manage to trylock you also go over the list, clean it and perform recovery. This list probably needs to be protected with per hive lock. I also think we can for now at least finish reviewing the first 2 patches and submit them since as I said before they are not dealing with this topic and fixing existing race conditions. If you are OK with that I can send for review the last iteration of the first 2 patches where I wait for the last fence in ring mirror list.
Andrey
I implemented HW reset avoidance including XGMI use case according to the plan i specified. Patch is attached but I can't test it yet due to XGMI regression in PSP which is supposed to be fixed soon. Please take a look.
Looks a bit too complicated on first glance. In general we should probably get away from handling a hive in any special way.
Yes, I guess i can do it the same way as the generic handling in amdgpu_device_gpu_recover - there is a list of devices to process which is of size 1 for non xgmi use case or more then 1 for XGMI.
Multiple timeout jobs in a hive are identical to multiple timeout jobs on different engines on a single device.
How about the following handling: 1. Add the timeout job to a list. 2. Try to grab a lock to handle the reset, if that doesn't work because there is already a reset underway return immediately. 3. Stop all schedulers on all affected devices including stopping the timeouts and detaching all callbacks. 4. Double check the list of timed out jobs, if all hw fences of all jobs are completed now we actually don't need to do anything.
What if all the jobs on the timed out list did complete but other job (or jobs) for which we removed the time out timer became hanged ? Wouldn't we miss a required reset in this case and wouldn't even have any indication of their hang ?
If the timeout triggers before we disable it we will have the job on the list of jobs which are hanging.
If we found that we don't reset and re-enable the timeout it will trigger a bit later and we can do the check again.
Thinking about this a bit more we actually don't need to change the handling in any way. The idea with the list is just overkill.
See when the we re-start the scheduler we will also re-arm the timeout, so it will trigger again. It is just a matter of clever timeout re-arming.
Christian.
This implies that hanged job might take in worst case twice (or more if there are recurrent resets) then the TO parameter value to trigger - is this OK ?
Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.
Now i got thinking about non hanged job in progress (job A) and let's say it's a long job , it just started executing but due to time out of another job (job B) on another (or this scheduler) it's parent cb got disconnected, we disarmed the tdr timer for the job's scheduler, meanwhile the timed out job did manage to complete before HW reset check and hence we skip HW reset, attach back the cb and rearm job's A tdr timer with a future value of few ms only - aren't we going to get false tdr triggered on job B now because we didn't let it enough time to run and complete ? I would prefer the other extreme of longer time for time out to trigger then false TDR. Optimally we would have per job timer and rearm to exactly the reminder of it's time out value - but we gave up on per job tdr work long ago. In general the more i think about it (correct me if I am wrong) I am less sure how much the optimization feature is useful - if job's time out did trigger what are the chances that the little more time we give it between beginning of tdr function and the time we do start the actual HW reset will be exactly what it needed to complete. Also, this is still not water proof as the job might complete and signal it's HW fence exactly after we checked for completion but before starting the HW reset code.
Andrey
By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?
Correct, yes.
About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI. What do you think ?
Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.
Christian.
But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.
That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.
I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does imply drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?
Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.
What we could do is to make it a timer instead and raise the work item from the device specific callback.
But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.
Christian.
Andrey
Andrey
5. Do the reset on all affected devices. 6. Drop the lock. 7. Add callbacks again and restart the schedulers.
I see your steps don't include flushing any tdr in progress from drm_sched_job_finish (or as I did it from amdgpu_job_free_cb) does it mean you don't think we need to flush in order to avoid freeing job while it still might be accessed from time out list ?
Andrey
Regards, Christian.
Andrey
Andrey
Andrey
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.orgmailto:amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[SNIP]
Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.
Now i got thinking about non hanged job in progress (job A) and let's say it's a long job , it just started executing but due to time out of another job (job B) on another (or this scheduler) it's parent cb got disconnected, we disarmed the tdr timer for the job's scheduler, meanwhile the timed out job did manage to complete before HW reset check and hence we skip HW reset, attach back the cb and rearm job's A tdr timer with a future value of few ms only - aren't we going to get false tdr triggered on job B now because we didn't let it enough time to run and complete ? I would prefer the other extreme of longer time for time out to trigger then false TDR. Optimally we would have per job timer and rearm to exactly the reminder of it's time out value - but we gave up on per job tdr work long ago.
Well we only re-arm the timeout with a shorter period if it already triggered once. If we just suspend the timeout then we should still use the longer period.
In general the more i think about it (correct me if I am wrong) I am less sure how much the optimization feature is useful - if job's time out did trigger what are the chances that the little more time we give it between beginning of tdr function and the time we do start the actual HW reset will be exactly what it needed to complete. Also, this is still not water proof as the job might complete and signal it's HW fence exactly after we checked for completion but before starting the HW reset code.
I don't see this as an optimization, but rather as mandatory for correct operation.
See without this we can run into issues because we execute jobs multiple times. That can still happen with this clean handling, but it is much more unlikely.
Christian.
Andrey
By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?
Correct, yes.
About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI. What do you think ?
Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.
Christian.
But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.
That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.
I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does imply drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?
Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.
What we could do is to make it a timer instead and raise the work item from the device specific callback.
But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.
Christian.
On 01/18/2019 04:25 AM, Koenig, Christian wrote:
[SNIP]
Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.
Now i got thinking about non hanged job in progress (job A) and let's say it's a long job , it just started executing but due to time out of another job (job B) on another (or this scheduler) it's parent cb got disconnected, we disarmed the tdr timer for the job's scheduler, meanwhile the timed out job did manage to complete before HW reset check and hence we skip HW reset, attach back the cb and rearm job's A tdr timer with a future value of few ms only - aren't we going to get false tdr triggered on job B now because we didn't let it enough time to run and complete ? I would prefer the other extreme of longer time for time out to trigger then false TDR. Optimally we would have per job timer and rearm to exactly the reminder of it's time out value - but we gave up on per job tdr work long ago.
Well we only re-arm the timeout with a shorter period if it already triggered once. If we just suspend the timeout then we should still use the longer period.
Can you explain more on this ? I don't get it.
Andrey
In general the more i think about it (correct me if I am wrong) I am less sure how much the optimization feature is useful - if job's time out did trigger what are the chances that the little more time we give it between beginning of tdr function and the time we do start the actual HW reset will be exactly what it needed to complete. Also, this is still not water proof as the job might complete and signal it's HW fence exactly after we checked for completion but before starting the HW reset code.
I don't see this as an optimization, but rather as mandatory for correct operation.
See without this we can run into issues because we execute jobs multiple times. That can still happen with this clean handling, but it is much more unlikely.
Christian.
Andrey
By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?
Correct, yes.
About flushing tdr jobs in progress from .free_job cb - looks like drm_sched_job_finish->cancel_delayed_work_sync is not enough, we still need to take care of flushing all sced->work_tdr for a device and for all devices in hive for XGMI. What do you think ?
Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.
Christian.
But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.
That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.
I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does imply drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?
Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.
What we could do is to make it a timer instead and raise the work item from the device specific callback.
But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.
Christian.
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
On 01/18/2019 04:25 AM, Koenig, Christian wrote:
[SNIP]
Re-arming the timeout should probably have a much reduced value when the job hasn't changed. E.g. something like a few ms.
Now i got thinking about non hanged job in progress (job A) and let's say it's a long job , it just started executing but due to time out of another job (job B) on another (or this scheduler) it's parent cb got disconnected, we disarmed the tdr timer for the job's scheduler, meanwhile the timed out job did manage to complete before HW reset check and hence we skip HW reset, attach back the cb and rearm job's A tdr timer with a future value of few ms only - aren't we going to get false tdr triggered on job B now because we didn't let it enough time to run and complete ? I would prefer the other extreme of longer time for time out to trigger then false TDR. Optimally we would have per job timer and rearm to exactly the reminder of it's time out value - but we gave up on per job tdr work long ago.
Well we only re-arm the timeout with a shorter period if it already triggered once. If we just suspend the timeout then we should still use the longer period.
Can you explain more on this ? I don't get it.
See drm_sched_job_timedout(), we re-arm the timeout at the end of the procedure.
We should change that and re-arm the timer with a much lower timeout if the job is still not finished.
Christian.
Andrey
In general the more i think about it (correct me if I am wrong) I am less sure how much the optimization feature is useful - if job's time out did trigger what are the chances that the little more time we give it between beginning of tdr function and the time we do start the actual HW reset will be exactly what it needed to complete. Also, this is still not water proof as the job might complete and signal it's HW fence exactly after we checked for completion but before starting the HW reset code.
I don't see this as an optimization, but rather as mandatory for correct operation.
See without this we can run into issues because we execute jobs multiple times. That can still happen with this clean handling, but it is much more unlikely.
Christian.
Andrey
By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?
Correct, yes.
> About flushing tdr jobs in progress from .free_job cb - looks like > drm_sched_job_finish->cancel_delayed_work_sync is not enough, we > still need to take care of flushing all sced->work_tdr for a > device and for all devices in hive for XGMI. > What do you think ? Why should that be necessary? We only wait for the delayed work to make sure that the job is not destroyed while dealing with it.
Christian.
But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.
That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.
I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does imply drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?
Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.
What we could do is to make it a timer instead and raise the work item from the device specific callback.
But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.
Christian.
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
On 01/18/2019 12:10 PM, Koenig, Christian wrote:
Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
On 01/18/2019 04:25 AM, Koenig, Christian wrote:
[SNIP]
> Re-arming the timeout should probably have a much reduced value > when the job hasn't changed. E.g. something like a few ms.
Now i got thinking about non hanged job in progress (job A) and let's say it's a long job , it just started executing but due to time out of another job (job B) on another (or this scheduler) it's parent cb got disconnected, we disarmed the tdr timer for the job's scheduler, meanwhile the timed out job did manage to complete before HW reset check and hence we skip HW reset, attach back the cb and rearm job's A tdr timer with a future value of few ms only - aren't we going to get false tdr triggered on job B now because we didn't let it enough time to run and complete ? I would prefer the other extreme of longer time for time out to trigger then false TDR. Optimally we would have per job timer and rearm to exactly the reminder of it's time out value - but we gave up on per job tdr work long ago.
Well we only re-arm the timeout with a shorter period if it already triggered once. If we just suspend the timeout then we should still use the longer period.
Can you explain more on this ? I don't get it.
See drm_sched_job_timedout(), we re-arm the timeout at the end of the procedure.
We should change that and re-arm the timer with a much lower timeout if the job is still not finished.
Christian.
I still don't see how this can fix the problem of of long job in progress triggering false tdr if no HW reset was done, but maybe I am missing other pieces you have in mind, I will finish the patch and send it and then we can be more specific based on the code.
Andrey
Andrey
In general the more i think about it (correct me if I am wrong) I am less sure how much the optimization feature is useful - if job's time out did trigger what are the chances that the little more time we give it between beginning of tdr function and the time we do start the actual HW reset will be exactly what it needed to complete. Also, this is still not water proof as the job might complete and signal it's HW fence exactly after we checked for completion but before starting the HW reset code.
I don't see this as an optimization, but rather as mandatory for correct operation.
See without this we can run into issues because we execute jobs multiple times. That can still happen with this clean handling, but it is much more unlikely.
Christian.
Andrey
By unchanged you mean when we didn't resubmit the job because of the optimized non HW reset, right ?
Correct, yes.
>> About flushing tdr jobs in progress from .free_job cb - looks like >> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we >> still need to take care of flushing all sced->work_tdr for a >> device and for all devices in hive for XGMI. >> What do you think ? > Why should that be necessary? We only wait for the delayed work to > make sure that the job is not destroyed while dealing with it. > > Christian. But we might not be waiting for the correct sched->work_tdr, we do the reset routine for all schedulers in a device accessing their jobs too and not only for the scheduler to which the job belongs. For XGMI not only that, we reset all the devices in the hive.
That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.
I was thinking, amdgpu driver is not even interested in allowing multiple sced->tdr to execute together - we have to serialize all of them anyway with the trylock mutex (even without XGMI), v3d in v3d_job_timedout seems also to reset all of his schedulers from the tdr work. Would it make sense to provide the sched->work_td as init parameter to scheduler (same one for all schedulers) so we can enforce serialization by disallowing more then 1 tdr work to execute in the same time ? Other drivers interested to do in parallel can provide unique sched->work_tdr per scheduler. This does imply drm_sched_job_timedout has to removed and delegated to specific driver implementation as probably other code dealing with sched->work_tdr... Maybe even move tdr handling to the driver all together ?
Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.
What we could do is to make it a timer instead and raise the work item from the device specific callback.
But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.
Christian.
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 18.01.19 um 18:34 schrieb Grodzovsky, Andrey:
On 01/18/2019 12:10 PM, Koenig, Christian wrote:
Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
On 01/18/2019 04:25 AM, Koenig, Christian wrote:
[SNIP]
>> Re-arming the timeout should probably have a much reduced value >> when the job hasn't changed. E.g. something like a few ms.
Now i got thinking about non hanged job in progress (job A) and let's say it's a long job , it just started executing but due to time out of another job (job B) on another (or this scheduler) it's parent cb got disconnected, we disarmed the tdr timer for the job's scheduler, meanwhile the timed out job did manage to complete before HW reset check and hence we skip HW reset, attach back the cb and rearm job's A tdr timer with a future value of few ms only - aren't we going to get false tdr triggered on job B now because we didn't let it enough time to run and complete ? I would prefer the other extreme of longer time for time out to trigger then false TDR. Optimally we would have per job timer and rearm to exactly the reminder of it's time out value - but we gave up on per job tdr work long ago.
Well we only re-arm the timeout with a shorter period if it already triggered once. If we just suspend the timeout then we should still use the longer period.
Can you explain more on this ? I don't get it.
See drm_sched_job_timedout(), we re-arm the timeout at the end of the procedure.
We should change that and re-arm the timer with a much lower timeout if the job is still not finished.
Christian.
I still don't see how this can fix the problem of of long job in progress triggering false tdr if no HW reset was done, but maybe I am missing other pieces you have in mind, I will finish the patch and send it and then we can be more specific based on the code.
Ok sounds good. We should probably discuss less on details and prototype a bit more.
Might be that I'm missing something here as well, so probably good to have some code to talk about things more directly.
Christian.
Andrey
Andrey
In general the more i think about it (correct me if I am wrong) I am less sure how much the optimization feature is useful - if job's time out did trigger what are the chances that the little more time we give it between beginning of tdr function and the time we do start the actual HW reset will be exactly what it needed to complete. Also, this is still not water proof as the job might complete and signal it's HW fence exactly after we checked for completion but before starting the HW reset code.
I don't see this as an optimization, but rather as mandatory for correct operation.
See without this we can run into issues because we execute jobs multiple times. That can still happen with this clean handling, but it is much more unlikely.
Christian.
Andrey
> By unchanged you mean when we didn't resubmit the job because of the > optimized non HW reset, right ? Correct, yes.
>>> About flushing tdr jobs in progress from .free_job cb - looks like >>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we >>> still need to take care of flushing all sced->work_tdr for a >>> device and for all devices in hive for XGMI. >>> What do you think ? >> Why should that be necessary? We only wait for the delayed work to >> make sure that the job is not destroyed while dealing with it. >> >> Christian. > But we might not be waiting for the correct sched->work_tdr, we do > the reset routine for all schedulers in a device accessing their > jobs too and not only for the scheduler to which the job belongs. > For XGMI not only that, we reset all the devices in the hive. That is harmless you only need to wait for the work_tdr of the current scheduler, not for all of them.
> I was thinking, amdgpu driver is not even interested in allowing > multiple sced->tdr to execute together - we have to serialize all of > them anyway with the trylock mutex (even without XGMI), v3d in > v3d_job_timedout seems also to reset all of his schedulers from the > tdr work. Would it make sense to provide the sched->work_td as init > parameter to scheduler (same one for all schedulers) so we can > enforce serialization by disallowing more then 1 tdr work to execute > in the same time ? Other drivers interested to do in parallel can > provide unique sched->work_tdr per scheduler. This does imply > drm_sched_job_timedout has to removed and delegated to specific > driver implementation as probably other code dealing with > sched->work_tdr... Maybe even move tdr handling to the driver all > together ? Yeah, I was thinking something similar. The problem with this approach is that a delayed work item can have only one delay, but for multiple engines we need multiple delays.
What we could do is to make it a timer instead and raise the work item from the device specific callback.
But that doesn't really saves us the stop all schedulers trouble, so it doesn't buy us much in the end if I see this correctly.
Christian.
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Attached series is the first 2 patches we already discussed about ring mirror list handling racing with all your comments fixed (still not committed). The third patch is a prototype based on the first 2 patches and on our discussion.
Please take a look.
Andrey
On 01/18/2019 01:32 PM, Koenig, Christian wrote:
Am 18.01.19 um 18:34 schrieb Grodzovsky, Andrey:
On 01/18/2019 12:10 PM, Koenig, Christian wrote:
Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
On 01/18/2019 04:25 AM, Koenig, Christian wrote:
[SNIP]
>>> Re-arming the timeout should probably have a much reduced value >>> when the job hasn't changed. E.g. something like a few ms. Now i got thinking about non hanged job in progress (job A) and let's say it's a long job , it just started executing but due to time out of another job (job B) on another (or this scheduler) it's parent cb got disconnected, we disarmed the tdr timer for the job's scheduler, meanwhile the timed out job did manage to complete before HW reset check and hence we skip HW reset, attach back the cb and rearm job's A tdr timer with a future value of few ms only - aren't we going to get false tdr triggered on job B now because we didn't let it enough time to run and complete ? I would prefer the other extreme of longer time for time out to trigger then false TDR. Optimally we would have per job timer and rearm to exactly the reminder of it's time out value - but we gave up on per job tdr work long ago.
Well we only re-arm the timeout with a shorter period if it already triggered once. If we just suspend the timeout then we should still use the longer period.
Can you explain more on this ? I don't get it.
See drm_sched_job_timedout(), we re-arm the timeout at the end of the procedure.
We should change that and re-arm the timer with a much lower timeout if the job is still not finished.
Christian.
I still don't see how this can fix the problem of of long job in progress triggering false tdr if no HW reset was done, but maybe I am missing other pieces you have in mind, I will finish the patch and send it and then we can be more specific based on the code.
Ok sounds good. We should probably discuss less on details and prototype a bit more.
Might be that I'm missing something here as well, so probably good to have some code to talk about things more directly.
Christian.
Andrey
Andrey
In general the more i think about it (correct me if I am wrong) I am less sure how much the optimization feature is useful - if job's time out did trigger what are the chances that the little more time we give it between beginning of tdr function and the time we do start the actual HW reset will be exactly what it needed to complete. Also, this is still not water proof as the job might complete and signal it's HW fence exactly after we checked for completion but before starting the HW reset code.
I don't see this as an optimization, but rather as mandatory for correct operation.
See without this we can run into issues because we execute jobs multiple times. That can still happen with this clean handling, but it is much more unlikely.
Christian.
Andrey
>> By unchanged you mean when we didn't resubmit the job because of the >> optimized non HW reset, right ? > Correct, yes. > >>>> About flushing tdr jobs in progress from .free_job cb - looks like >>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we >>>> still need to take care of flushing all sced->work_tdr for a >>>> device and for all devices in hive for XGMI. >>>> What do you think ? >>> Why should that be necessary? We only wait for the delayed work to >>> make sure that the job is not destroyed while dealing with it. >>> >>> Christian. >> But we might not be waiting for the correct sched->work_tdr, we do >> the reset routine for all schedulers in a device accessing their >> jobs too and not only for the scheduler to which the job belongs. >> For XGMI not only that, we reset all the devices in the hive. > That is harmless you only need to wait for the work_tdr of the > current scheduler, not for all of them. > >> I was thinking, amdgpu driver is not even interested in allowing >> multiple sced->tdr to execute together - we have to serialize all of >> them anyway with the trylock mutex (even without XGMI), v3d in >> v3d_job_timedout seems also to reset all of his schedulers from the >> tdr work. Would it make sense to provide the sched->work_td as init >> parameter to scheduler (same one for all schedulers) so we can >> enforce serialization by disallowing more then 1 tdr work to execute >> in the same time ? Other drivers interested to do in parallel can >> provide unique sched->work_tdr per scheduler. This does imply >> drm_sched_job_timedout has to removed and delegated to specific >> driver implementation as probably other code dealing with >> sched->work_tdr... Maybe even move tdr handling to the driver all >> together ? > Yeah, I was thinking something similar. The problem with this > approach is that a delayed work item can have only one delay, but for > multiple engines we need multiple delays. > > What we could do is to make it a timer instead and raise the work > item from the device specific callback. > > But that doesn't really saves us the stop all schedulers trouble, so > it doesn't buy us much in the end if I see this correctly. > > Christian.
amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
I see a few cleanups on Patch #3 which actually belong in patch #1:
+void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
The "bad" job parameter actually isn't used any more, isn't it?
+retry_wait:
Not used any more.
But apart from that at least patch #1 and #2 look like they can have my rb now.
Patch #3 looks also like it should work after a bit of polishing.
Thanks, Christian.
Am 18.01.19 um 20:15 schrieb Grodzovsky, Andrey:
Attached series is the first 2 patches we already discussed about ring mirror list handling racing with all your comments fixed (still not committed). The third patch is a prototype based on the first 2 patches and on our discussion.
Please take a look.
Andrey
On 01/18/2019 01:32 PM, Koenig, Christian wrote:
Am 18.01.19 um 18:34 schrieb Grodzovsky, Andrey:
On 01/18/2019 12:10 PM, Koenig, Christian wrote:
Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
On 01/18/2019 04:25 AM, Koenig, Christian wrote:
[SNIP] >>>> Re-arming the timeout should probably have a much reduced value >>>> when the job hasn't changed. E.g. something like a few ms. > Now i got thinking about non hanged job in progress (job A) and let's > say it's a long job , it just started executing but due to time out of > another job (job B) on another (or this scheduler) it's parent cb got > disconnected, we disarmed the tdr timer for the job's scheduler, > meanwhile the timed out job did manage to complete before HW reset > check and hence we skip HW reset, attach back the cb and rearm job's A > tdr timer with a future value of few ms only - aren't we going to get > false tdr triggered on job B now because we didn't let it enough time > to run and complete ? I would prefer the other extreme of longer time > for time out to trigger then false TDR. Optimally we would have per > job timer and rearm to exactly the reminder of it's time out value - > but we gave up on per job tdr work long ago. Well we only re-arm the timeout with a shorter period if it already triggered once. If we just suspend the timeout then we should still use the longer period.
Can you explain more on this ? I don't get it.
See drm_sched_job_timedout(), we re-arm the timeout at the end of the procedure.
We should change that and re-arm the timer with a much lower timeout if the job is still not finished.
Christian.
I still don't see how this can fix the problem of of long job in progress triggering false tdr if no HW reset was done, but maybe I am missing other pieces you have in mind, I will finish the patch and send it and then we can be more specific based on the code.
Ok sounds good. We should probably discuss less on details and prototype a bit more.
Might be that I'm missing something here as well, so probably good to have some code to talk about things more directly.
Christian.
Andrey
Andrey
> In general the more i think about it (correct me if I am wrong) I am > less sure how much the optimization feature is useful - if job's time > out did trigger what are the chances that the little more time we give > it between beginning of tdr function and the time we do start the > actual HW reset will be exactly what it needed to complete. Also, this > is still not water proof as the job might complete and signal it's HW > fence exactly after we checked for completion but before starting the > HW reset code. I don't see this as an optimization, but rather as mandatory for correct operation.
See without this we can run into issues because we execute jobs multiple times. That can still happen with this clean handling, but it is much more unlikely.
Christian.
> Andrey > >>> By unchanged you mean when we didn't resubmit the job because of the >>> optimized non HW reset, right ? >> Correct, yes. >> >>>>> About flushing tdr jobs in progress from .free_job cb - looks like >>>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we >>>>> still need to take care of flushing all sced->work_tdr for a >>>>> device and for all devices in hive for XGMI. >>>>> What do you think ? >>>> Why should that be necessary? We only wait for the delayed work to >>>> make sure that the job is not destroyed while dealing with it. >>>> >>>> Christian. >>> But we might not be waiting for the correct sched->work_tdr, we do >>> the reset routine for all schedulers in a device accessing their >>> jobs too and not only for the scheduler to which the job belongs. >>> For XGMI not only that, we reset all the devices in the hive. >> That is harmless you only need to wait for the work_tdr of the >> current scheduler, not for all of them. >> >>> I was thinking, amdgpu driver is not even interested in allowing >>> multiple sced->tdr to execute together - we have to serialize all of >>> them anyway with the trylock mutex (even without XGMI), v3d in >>> v3d_job_timedout seems also to reset all of his schedulers from the >>> tdr work. Would it make sense to provide the sched->work_td as init >>> parameter to scheduler (same one for all schedulers) so we can >>> enforce serialization by disallowing more then 1 tdr work to execute >>> in the same time ? Other drivers interested to do in parallel can >>> provide unique sched->work_tdr per scheduler. This does imply >>> drm_sched_job_timedout has to removed and delegated to specific >>> driver implementation as probably other code dealing with >>> sched->work_tdr... Maybe even move tdr handling to the driver all >>> together ? >> Yeah, I was thinking something similar. The problem with this >> approach is that a delayed work item can have only one delay, but for >> multiple engines we need multiple delays. >> >> What we could do is to make it a timer instead and raise the work >> item from the device specific callback. >> >> But that doesn't really saves us the stop all schedulers trouble, so >> it doesn't buy us much in the end if I see this correctly. >> >> Christian. _______________________________________________ amd-gfx mailing list amd-gfx@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/amd-gfx
OK, I will update patches 1 and 2 and given your RBs push them since they fix some races. I will then update and test patch 3 on some basic scenarios and will send it for separate review where I might put a TODO comment in code with my objections regarding long jobs form our discussion so you can see and reply on that.
Andrey
On 01/24/2019 06:34 AM, Koenig, Christian wrote:
I see a few cleanups on Patch #3 which actually belong in patch #1:
+void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
The "bad" job parameter actually isn't used any more, isn't it?
+retry_wait:
Not used any more.
But apart from that at least patch #1 and #2 look like they can have my rb now.
Patch #3 looks also like it should work after a bit of polishing.
Thanks, Christian.
Am 18.01.19 um 20:15 schrieb Grodzovsky, Andrey:
Attached series is the first 2 patches we already discussed about ring mirror list handling racing with all your comments fixed (still not committed). The third patch is a prototype based on the first 2 patches and on our discussion.
Please take a look.
Andrey
On 01/18/2019 01:32 PM, Koenig, Christian wrote:
Am 18.01.19 um 18:34 schrieb Grodzovsky, Andrey:
On 01/18/2019 12:10 PM, Koenig, Christian wrote:
Am 18.01.19 um 16:21 schrieb Grodzovsky, Andrey:
On 01/18/2019 04:25 AM, Koenig, Christian wrote: > [SNIP] >>>>> Re-arming the timeout should probably have a much reduced value >>>>> when the job hasn't changed. E.g. something like a few ms. >> Now i got thinking about non hanged job in progress (job A) and let's >> say it's a long job , it just started executing but due to time out of >> another job (job B) on another (or this scheduler) it's parent cb got >> disconnected, we disarmed the tdr timer for the job's scheduler, >> meanwhile the timed out job did manage to complete before HW reset >> check and hence we skip HW reset, attach back the cb and rearm job's A >> tdr timer with a future value of few ms only - aren't we going to get >> false tdr triggered on job B now because we didn't let it enough time >> to run and complete ? I would prefer the other extreme of longer time >> for time out to trigger then false TDR. Optimally we would have per >> job timer and rearm to exactly the reminder of it's time out value - >> but we gave up on per job tdr work long ago. > Well we only re-arm the timeout with a shorter period if it already > triggered once. If we just suspend the timeout then we should still use > the longer period. Can you explain more on this ? I don't get it.
See drm_sched_job_timedout(), we re-arm the timeout at the end of the procedure.
We should change that and re-arm the timer with a much lower timeout if the job is still not finished.
Christian.
I still don't see how this can fix the problem of of long job in progress triggering false tdr if no HW reset was done, but maybe I am missing other pieces you have in mind, I will finish the patch and send it and then we can be more specific based on the code.
Ok sounds good. We should probably discuss less on details and prototype a bit more.
Might be that I'm missing something here as well, so probably good to have some code to talk about things more directly.
Christian.
Andrey
Andrey
>> In general the more i think about it (correct me if I am wrong) I am >> less sure how much the optimization feature is useful - if job's time >> out did trigger what are the chances that the little more time we give >> it between beginning of tdr function and the time we do start the >> actual HW reset will be exactly what it needed to complete. Also, this >> is still not water proof as the job might complete and signal it's HW >> fence exactly after we checked for completion but before starting the >> HW reset code. > I don't see this as an optimization, but rather as mandatory for correct > operation. > > See without this we can run into issues because we execute jobs multiple > times. That can still happen with this clean handling, but it is much > more unlikely. > > Christian. > >> Andrey >> >>>> By unchanged you mean when we didn't resubmit the job because of the >>>> optimized non HW reset, right ? >>> Correct, yes. >>> >>>>>> About flushing tdr jobs in progress from .free_job cb - looks like >>>>>> drm_sched_job_finish->cancel_delayed_work_sync is not enough, we >>>>>> still need to take care of flushing all sced->work_tdr for a >>>>>> device and for all devices in hive for XGMI. >>>>>> What do you think ? >>>>> Why should that be necessary? We only wait for the delayed work to >>>>> make sure that the job is not destroyed while dealing with it. >>>>> >>>>> Christian. >>>> But we might not be waiting for the correct sched->work_tdr, we do >>>> the reset routine for all schedulers in a device accessing their >>>> jobs too and not only for the scheduler to which the job belongs. >>>> For XGMI not only that, we reset all the devices in the hive. >>> That is harmless you only need to wait for the work_tdr of the >>> current scheduler, not for all of them. >>> >>>> I was thinking, amdgpu driver is not even interested in allowing >>>> multiple sced->tdr to execute together - we have to serialize all of >>>> them anyway with the trylock mutex (even without XGMI), v3d in >>>> v3d_job_timedout seems also to reset all of his schedulers from the >>>> tdr work. Would it make sense to provide the sched->work_td as init >>>> parameter to scheduler (same one for all schedulers) so we can >>>> enforce serialization by disallowing more then 1 tdr work to execute >>>> in the same time ? Other drivers interested to do in parallel can >>>> provide unique sched->work_tdr per scheduler. This does imply >>>> drm_sched_job_timedout has to removed and delegated to specific >>>> driver implementation as probably other code dealing with >>>> sched->work_tdr... Maybe even move tdr handling to the driver all >>>> together ? >>> Yeah, I was thinking something similar. The problem with this >>> approach is that a delayed work item can have only one delay, but for >>> multiple engines we need multiple delays. >>> >>> What we could do is to make it a timer instead and raise the work >>> item from the device specific callback. >>> >>> But that doesn't really saves us the stop all schedulers trouble, so >>> it doesn't buy us much in the end if I see this correctly. >>> >>> Christian. > _______________________________________________ > amd-gfx mailing list > amd-gfx@lists.freedesktop.org > https://lists.freedesktop.org/mailman/listinfo/amd-gfx
dri-devel@lists.freedesktop.org
-
Andrey Grodzovsky -
Christian König -
Grodzovsky, Andrey
-
Koenig, Christian