On 28/06/2021 08:42, Boris Brezillon wrote:
From: Steven Price steven.price@arm.com
The hardware has a set of '_NEXT' registers that can hold a second job while the first is executing. Make use of these registers to enqueue a second job per slot.
v3:
- Fix the done/err job dequeuing logic to get a valid active state
- Only enable the second slot on GPUs supporting jobchain disambiguation
- Split interrupt handling in sub-functions
Signed-off-by: Steven Price steven.price@arm.com Signed-off-by: Boris Brezillon boris.brezillon@collabora.com
One nit below...
drivers/gpu/drm/panfrost/panfrost_device.h | 2 +- drivers/gpu/drm/panfrost/panfrost_job.c | 468 +++++++++++++++------ 2 files changed, 351 insertions(+), 119 deletions(-)
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h index d91f71366214..81f81fc8650e 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.h +++ b/drivers/gpu/drm/panfrost/panfrost_device.h @@ -101,7 +101,7 @@ struct panfrost_device {
struct panfrost_job_slot *js;
- struct panfrost_job *jobs[NUM_JOB_SLOTS];
struct panfrost_job *jobs[NUM_JOB_SLOTS][2]; struct list_head scheduled_jobs;
struct panfrost_perfcnt *perfcnt;
diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c index 979108dbc323..b965669a1fed 100644 --- a/drivers/gpu/drm/panfrost/panfrost_job.c +++ b/drivers/gpu/drm/panfrost/panfrost_job.c @@ -4,6 +4,7 @@ #include <linux/delay.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/iopoll.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/dma-resv.h> @@ -140,9 +141,52 @@ static void panfrost_job_write_affinity(struct panfrost_device *pfdev, job_write(pfdev, JS_AFFINITY_NEXT_HI(js), affinity >> 32); }
+static u32 +panfrost_get_job_chain_flag(const struct panfrost_job *job) +{
- struct panfrost_fence *f = to_panfrost_fence(job->done_fence);
- if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
return 0;- return (f->seqno & 1) ? JS_CONFIG_JOB_CHAIN_FLAG : 0;
+}
+static struct panfrost_job * +panfrost_dequeue_job(struct panfrost_device *pfdev, int slot) +{
- struct panfrost_job *job = pfdev->jobs[slot][0];
- WARN_ON(!job);
- pfdev->jobs[slot][0] = pfdev->jobs[slot][1];
- pfdev->jobs[slot][1] = NULL;
- return job;
+}
+static unsigned int +panfrost_enqueue_job(struct panfrost_device *pfdev, int slot,
struct panfrost_job *job)+{
- if (WARN_ON(!job))
return 0;- if (!pfdev->jobs[slot][0]) {
pfdev->jobs[slot][0] = job;return 0;- }
- WARN_ON(pfdev->jobs[slot][1]);
- pfdev->jobs[slot][1] = job;
- WARN_ON(panfrost_get_job_chain_flag(job) ==
panfrost_get_job_chain_flag(pfdev->jobs[slot][0]));- return 1;
+}
static void panfrost_job_hw_submit(struct panfrost_job *job, int js) { struct panfrost_device *pfdev = job->pfdev;
- unsigned int subslot; u32 cfg; u64 jc_head = job->jc; int ret;
@@ -168,7 +212,8 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js) * start */ cfg |= JS_CONFIG_THREAD_PRI(8) | JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE |
JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE;
JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE |panfrost_get_job_chain_flag(job);if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION)) cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
@@ -182,10 +227,17 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js) job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);
/* GO ! */
dev_dbg(pfdev->dev, "JS: Submitting atom %p to js[%d] with head=0x%llx",
job, js, jc_head);job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);
- spin_lock(&pfdev->js->job_lock);
- subslot = panfrost_enqueue_job(pfdev, js, job);
- /* Don't queue the job if a reset is in progress */
- if (!atomic_read(&pfdev->reset.pending)) {
job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);dev_dbg(pfdev->dev,"JS: Submitting atom %p to js[%d][%d] with head=0x%llx AS %d",job, js, subslot, jc_head, cfg & 0xf);- }
- spin_unlock(&pfdev->js->job_lock);
}
static void panfrost_acquire_object_fences(struct drm_gem_object **bos, @@ -343,7 +395,11 @@ static struct dma_fence *panfrost_job_run(struct drm_sched_job *sched_job) if (unlikely(job->base.s_fence->finished.error)) return NULL;
- pfdev->jobs[slot] = job;
/* Nothing to execute: can happen if the job has finished while
* we were resetting the GPU.*/if (!job->jc)
return NULL;fence = panfrost_fence_create(pfdev, slot); if (IS_ERR(fence))
@@ -371,11 +427,218 @@ void panfrost_job_enable_interrupts(struct panfrost_device *pfdev) job_write(pfdev, JOB_INT_MASK, irq_mask); }
-static void panfrost_reset(struct panfrost_device *pfdev,
struct drm_sched_job *bad)+static void panfrost_job_handle_err(struct panfrost_device *pfdev,
struct panfrost_job *job,unsigned int js){
- unsigned int i;
- u32 js_status = job_read(pfdev, JS_STATUS(js));
- const char *exception_name = panfrost_exception_name(js_status);
- bool signal_fence = true;
- if (js_status < DRM_PANFROST_EXCEPTION_JOB_CONFIG_FAULT) {
dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x",js, exception_name,job_read(pfdev, JS_HEAD_LO(js)),job_read(pfdev, JS_TAIL_LO(js)));- } else {
dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",js, exception_name,job_read(pfdev, JS_HEAD_LO(js)),job_read(pfdev, JS_TAIL_LO(js)));- }
- if (js_status == DRM_PANFROST_EXCEPTION_STOPPED) {
/* Update the job head so we can resume */job->jc = job_read(pfdev, JS_TAIL_LO(js)) |((u64)job_read(pfdev, JS_TAIL_HI(js)) << 32);/* The job will be resumed, don't signal the fence */signal_fence = false;- } else if (js_status == DRM_PANFROST_EXCEPTION_TERMINATED) {
/* Job has been hard-stopped, flag it as canceled */dma_fence_set_error(job->done_fence, -ECANCELED);job->jc = 0;- } else if (js_status >= DRM_PANFROST_EXCEPTION_JOB_CONFIG_FAULT) {
/* We might want to provide finer-grained error code based on* the exception type, but unconditionally setting to EINVAL* is good enough for now.*/dma_fence_set_error(job->done_fence, -EINVAL);job->jc = 0;- }
- panfrost_mmu_as_put(pfdev, job->file_priv->mmu);
- panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
- if (signal_fence)
dma_fence_signal_locked(job->done_fence);- pm_runtime_put_autosuspend(pfdev->dev);
- if (panfrost_exception_needs_reset(pfdev, js_status)) {
atomic_set(&pfdev->reset.pending, 1);drm_sched_fault(&pfdev->js->queue[js].sched);- }
+}
+static void panfrost_job_handle_done(struct panfrost_device *pfdev,
struct panfrost_job *job)+{
- /* Set ->jc to 0 to avoid re-submitting an already finished job (can
* happen when we receive the DONE interrupt while doing a GPU reset).*/- job->jc = 0;
- panfrost_mmu_as_put(pfdev, job->file_priv->mmu);
- panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
- dma_fence_signal_locked(job->done_fence);
- pm_runtime_put_autosuspend(pfdev->dev);
+}
+static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status) +{
- struct panfrost_job *done[NUM_JOB_SLOTS][2] = {};
- struct panfrost_job *failed[NUM_JOB_SLOTS] = {};
- u32 js_state = 0, js_events = 0;
- unsigned int i, j;
- /* First we collect all failed/done jobs. */
- while (status) {
u32 js_state_mask = 0;for (j = 0; j < NUM_JOB_SLOTS; j++) {if (status & MK_JS_MASK(j))js_state_mask |= MK_JS_MASK(j);if (status & JOB_INT_MASK_DONE(j)) {if (done[j][0])done[j][1] = panfrost_dequeue_job(pfdev, j);elsedone[j][0] = panfrost_dequeue_job(pfdev, j);}if (status & JOB_INT_MASK_ERR(j)) {/* Cancel the next submission. Will be submitted* after we're done handling this failure if* there's no reset pending.*/job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);failed[j] = panfrost_dequeue_job(pfdev, j);}}/* JS_STATE is sampled when JOB_INT_CLEAR is written.* For each BIT(slot) or BIT(slot + 16) bit written to* JOB_INT_CLEAR, the corresponding bits in JS_STATE* (BIT(slot) and BIT(slot + 16)) are updated, but this* is racy. If we only have one job done at the time we* read JOB_INT_RAWSTAT but the second job fails before we* clear the status, we end up with a status containing* only the DONE bit and consider both jobs as DONE since* JS_STATE reports both NEXT and CURRENT as inactive.* To prevent that, let's repeat this clear+read steps* until status is 0.*/job_write(pfdev, JOB_INT_CLEAR, status);js_state &= ~js_state_mask;js_state |= job_read(pfdev, JOB_INT_JS_STATE) & js_state_mask;js_events |= status;status = job_read(pfdev, JOB_INT_RAWSTAT);- }
- /* Then we handle the dequeued jobs. */
- for (j = 0; j < NUM_JOB_SLOTS; j++) {
if (!(js_events & MK_JS_MASK(j)))continue;if (failed[j]) {panfrost_job_handle_err(pfdev, failed[j], j);} else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) {/* When the current job doesn't fail, the JM dequeues* the next job without waiting for an ACK, this means* we can have 2 jobs dequeued and only catch the* interrupt when the second one is done. If both slots* are inactive, but one job remains in pfdev->jobs[j],* consider it done. Of course that doesn't apply if a* failure happened since we cancelled execution of the* job in _NEXT (see above).*/if (WARN_ON(!done[j][0]))done[j][0] = panfrost_dequeue_job(pfdev, j);elsedone[j][1] = panfrost_dequeue_job(pfdev, j);}for (i = 0; i < ARRAY_SIZE(done[0]) && done[j][i]; i++)panfrost_job_handle_done(pfdev, done[j][i]);- }
- /* And finally we requeue jobs that were waiting in the second slot
* and have been stopped if we detected a failure on the first slot.*/- for (j = 0; j < NUM_JOB_SLOTS; j++) {
if (!(js_events & MK_JS_MASK(j)))continue;if (!failed[j] || !pfdev->jobs[j][0])continue;if (pfdev->jobs[j][0]->jc == 0) {/* The job was cancelled, signal the fence now */struct panfrost_job *canceled = panfrost_dequeue_job(pfdev, j);dma_fence_set_error(canceled->done_fence, -ECANCELED);panfrost_job_handle_done(pfdev, canceled);} else if (!atomic_read(&pfdev->reset.pending)) {/* Requeue the job we removed if no reset is pending */job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_START);}- }
+}
+static void panfrost_job_handle_irqs(struct panfrost_device *pfdev) +{
- u32 status = job_read(pfdev, JOB_INT_RAWSTAT);
- while (status) {
pm_runtime_mark_last_busy(pfdev->dev);spin_lock(&pfdev->js->job_lock);panfrost_job_handle_irq(pfdev, status);spin_unlock(&pfdev->js->job_lock);status = job_read(pfdev, JOB_INT_RAWSTAT);- }
+}
+static u32 panfrost_active_slots(struct panfrost_device *pfdev,
u32 *js_state_mask, u32 js_state)+{
- u32 rawstat;
- if (!(js_state & *js_state_mask))
return 0;- rawstat = job_read(pfdev, JOB_INT_RAWSTAT);
- if (rawstat) {
unsigned int i;for (i = 0; i < NUM_JOB_SLOTS; i++) {if (rawstat & MK_JS_MASK(i))*js_state_mask &= ~MK_JS_MASK(i);}- }
- return js_state & *js_state_mask;
+}
+static void +panfrost_reset(struct panfrost_device *pfdev,
struct drm_sched_job *bad)+{
u32 js_state, js_state_mask = 0xffffffff;
unsigned int i, j; bool cookie;
int ret;
if (!atomic_read(&pfdev->reset.pending)) return;
@@ -407,21 +670,46 @@ static void panfrost_reset(struct panfrost_device *pfdev, job_write(pfdev, JOB_INT_MASK, 0); synchronize_irq(pfdev->js->irq);
- /* Schedulers are stopped and interrupts are masked+flushed, we don't
* need to protect the 'evict unfinished jobs' lock with the job_lock.
- for (i = 0; i < NUM_JOB_SLOTS; i++) {
/* Cancel the next job and soft-stop the running job. */job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);job_write(pfdev, JS_COMMAND(i), JS_COMMAND_SOFT_STOP);- }
- /* Wait at most 10ms for soft-stops to complete */
- ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state,
!panfrost_active_slots(pfdev, &js_state_mask, js_state),10, 10000);- if (ret)
dev_err(pfdev->dev, "Soft-stop failed\n");- /* Handle the remaining interrupts before we reset. */
- panfrost_job_handle_irqs(pfdev);
- /* Remaining interrupts have been handled, but we might still have
* stuck jobs. Let's make sure the PM counters stay balanced by* manually calling pm_runtime_put_noidle() and* panfrost_devfreq_record_idle() for each stuck job.
if (pfdev->jobs[i]) {
for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) { pm_runtime_put_noidle(pfdev->dev); panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
pfdev->jobs[i] = NULL;
memset(pfdev->jobs, 0, sizeof(pfdev->jobs)); spin_unlock(&pfdev->js->job_lock);
/* Proceed with reset now. */ panfrost_device_reset(pfdev);
/* panfrost_device_reset() unmasks job interrupts, but we want to
* keep them masked a bit longer.*/job_write(pfdev, JOB_INT_MASK, 0);
/* GPU has been reset, we can cancel timeout/fault work that may have
- been queued in the meantime and clear the reset pending bit.
*/
@@ -429,7 +717,6 @@ static void panfrost_reset(struct panfrost_device *pfdev, for (i = 0; i < NUM_JOB_SLOTS; i++) cancel_delayed_work(&pfdev->js->queue[i].sched.work_tdr);
- /* Now resubmit jobs that were previously queued but didn't have a
- chance to finish.
- FIXME: We temporarily get out of the DMA fence signalling section
@@ -442,9 +729,15 @@ static void panfrost_reset(struct panfrost_device *pfdev, drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched); cookie = dma_fence_begin_signalling();
/* Restart the schedulers */ for (i = 0; i < NUM_JOB_SLOTS; i++) drm_sched_start(&pfdev->js->queue[i].sched, true);
/* Re-enable job interrupts now that everything has been restarted. */
job_write(pfdev, JOB_INT_MASK,
GENMASK(16 + NUM_JOB_SLOTS - 1, 16) |GENMASK(NUM_JOB_SLOTS - 1, 0));dma_fence_end_signalling(cookie);
}
@@ -476,6 +769,14 @@ static enum drm_gpu_sched_stat panfrost_job_timedout(struct drm_sched_job return DRM_GPU_SCHED_STAT_NOMINAL; }
+static void panfrost_reset_work(struct work_struct *work) +{
- struct panfrost_device *pfdev;
- pfdev = container_of(work, struct panfrost_device, reset.work);
- panfrost_reset(pfdev, NULL);
+}
static const struct drm_sched_backend_ops panfrost_sched_ops = { .dependency = panfrost_job_dependency, .run_job = panfrost_job_run, @@ -483,100 +784,11 @@ static const struct drm_sched_backend_ops panfrost_sched_ops = { .free_job = panfrost_job_free };
-static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status) -{
- int j;
- dev_dbg(pfdev->dev, "jobslot irq status=%x\n", status);
- for (j = 0; status; j++) {
u32 mask = MK_JS_MASK(j);if (!(status & mask))continue;job_write(pfdev, JOB_INT_CLEAR, mask);if (status & JOB_INT_MASK_ERR(j)) {u32 js_status = job_read(pfdev, JS_STATUS(j));const char *exception_name = panfrost_exception_name(js_status);job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);if (js_status < DRM_PANFROST_EXCEPTION_JOB_CONFIG_FAULT) {dev_dbg(pfdev->dev, "js interrupt, js=%d, status=%s, head=0x%x, tail=0x%x",j, exception_name,job_read(pfdev, JS_HEAD_LO(j)),job_read(pfdev, JS_TAIL_LO(j)));} else {dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",j, exception_name,job_read(pfdev, JS_HEAD_LO(j)),job_read(pfdev, JS_TAIL_LO(j)));}/* If we need a reset, signal it to the timeout* handler, otherwise, update the fence error field and* signal the job fence.*/if (panfrost_exception_needs_reset(pfdev, js_status)) {drm_sched_fault(&pfdev->js->queue[j].sched);} else {int error = 0;if (js_status == DRM_PANFROST_EXCEPTION_TERMINATED)error = -ECANCELED;else if (js_status >= DRM_PANFROST_EXCEPTION_JOB_CONFIG_FAULT)error = -EINVAL;if (error)dma_fence_set_error(pfdev->jobs[j]->done_fence, error);status |= JOB_INT_MASK_DONE(j);}}if (status & JOB_INT_MASK_DONE(j)) {struct panfrost_job *job;job = pfdev->jobs[j];/* The only reason this job could be NULL is if the* job IRQ handler is called just after the* in-flight job eviction in the reset path, and* this shouldn't happen because the job IRQ has* been masked and synchronized when this eviction* happens.*/WARN_ON(!job);if (job) {pfdev->jobs[j] = NULL;panfrost_mmu_as_put(pfdev, job->file_priv->mmu);panfrost_devfreq_record_idle(&pfdev->pfdevfreq);dma_fence_signal_locked(job->done_fence);pm_runtime_put_autosuspend(pfdev->dev);}}status &= ~mask;- }
-}
static irqreturn_t panfrost_job_irq_handler_thread(int irq, void *data) { struct panfrost_device *pfdev = data;
- u32 status = job_read(pfdev, JOB_INT_RAWSTAT);
- while (status) {
pm_runtime_mark_last_busy(pfdev->dev);spin_lock(&pfdev->js->job_lock);panfrost_job_handle_irq(pfdev, status);spin_unlock(&pfdev->js->job_lock);status = job_read(pfdev, JOB_INT_RAWSTAT);- }
- panfrost_job_handle_irqs(pfdev); job_write(pfdev, JOB_INT_MASK, GENMASK(16 + NUM_JOB_SLOTS - 1, 16) | GENMASK(NUM_JOB_SLOTS - 1, 0));
@@ -595,26 +807,24 @@ static irqreturn_t panfrost_job_irq_handler(int irq, void *data) return IRQ_WAKE_THREAD; }
-static void panfrost_reset_work(struct work_struct *work) -{
- struct panfrost_device *pfdev = container_of(work,
struct panfrost_device,reset.work);- panfrost_reset(pfdev, NULL);
-}
int panfrost_job_init(struct panfrost_device *pfdev) { struct panfrost_job_slot *js;
- unsigned int nslots = 2; int ret, j;
- INIT_WORK(&pfdev->reset.work, panfrost_reset_work);
- /* All GPUs have twnslotsueue, but without jobchain
^^^^^^^^^^^^ It's a made up word ;)
Also the use of "slot" here is confusing because it's referring to hw submission entries in the call to drm_sched_init() and not the "job slots" that the HW has (Mali GPUs have 3 slots).
Thanks,
Steve
* disambiguation stopping the right job in the close path is tricky,* so let's just advertize one slot in that case.*/if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION))
nslots = 1;pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL); if (!js) return -ENOMEM;
INIT_WORK(&pfdev->reset.work, panfrost_reset_work); spin_lock_init(&js->job_lock);
js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");
@@ -640,7 +850,7 @@ int panfrost_job_init(struct panfrost_device *pfdev)
ret = drm_sched_init(&js->queue[j].sched, &panfrost_sched_ops,
1, 0,
nslots, 0, msecs_to_jiffies(JOB_TIMEOUT_MS), pfdev->reset.wq, NULL, "pan_js");@@ -707,12 +917,34 @@ void panfrost_job_close(struct panfrost_file_priv *panfrost_priv) spin_lock(&pfdev->js->job_lock); for (i = 0; i < NUM_JOB_SLOTS; i++) { struct drm_sched_entity *entity = &panfrost_priv->sched_entity[i];
struct panfrost_job *job = pfdev->jobs[i];
int j;
if (!job || job->base.entity != entity)continue;
for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) {struct panfrost_job *job = pfdev->jobs[i][j];u32 cmd;
job_write(pfdev, JS_COMMAND(i), JS_COMMAND_HARD_STOP);
if (!job || job->base.entity != entity)continue;if (j == 1) {/* Try to cancel the job before it starts */job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP);/* Reset the job head so it doesn't get restarted if* the job in the first slot failed.*/job->jc = 0;}if (panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {cmd = panfrost_get_job_chain_flag(job) ?JS_COMMAND_HARD_STOP_1 :JS_COMMAND_HARD_STOP_0;} else {cmd = JS_COMMAND_HARD_STOP;}job_write(pfdev, JS_COMMAND(i), cmd);}}