From: David Yat Sin david.yatsin@amd.com
When re-creating queues during CRIU restore, restore the queue with the same doorbell id value used during CRIU dump.
Signed-off-by: David Yat Sin david.yatsin@amd.com Change-Id: I6a79de1f8c760d5a9d28e7951740296f2f8796a7 --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 1 + .../drm/amd/amdkfd/kfd_device_queue_manager.c | 58 +++++++++++++------ drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 1 + 3 files changed, 41 insertions(+), 19 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index afcbdae436fa..130ab100abb2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -2153,6 +2153,7 @@ int criu_restore_queue(struct kfd_process *p,
qrd->qid = q_bucket->q_id; qrd->sdma_id = q_bucket->sdma_id; + qrd->doorbell_id = q_bucket->doorbell_id;
ret = pqm_create_queue(&p->pqm, dev, NULL, &qp, &queue_id, qrd, NULL); if (ret) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index cabdfbacce37..56250c0b5ca0 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -153,7 +153,11 @@ static void decrement_queue_count(struct device_queue_manager *dqm, dqm->active_cp_queue_count--; }
-static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q) +/* + * Allocate a doorbell ID to this queue. + * If doorbell_id is passed in, make sure requested ID is valid then allocate it. + */ +static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q, uint32_t const *restore_id) { struct kfd_dev *dev = qpd->dqm->dev;
@@ -161,6 +165,9 @@ static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q) /* On pre-SOC15 chips we need to use the queue ID to * preserve the user mode ABI. */ + if (restore_id && *restore_id != q->properties.queue_id) + return -EINVAL; + q->doorbell_id = q->properties.queue_id; } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { @@ -169,25 +176,38 @@ static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q) * The doobell index distance between RLC (2*i) and (2*i+1) * for a SDMA engine is 512. */ - uint32_t *idx_offset = - dev->shared_resources.sdma_doorbell_idx;
- q->doorbell_id = idx_offset[q->properties.sdma_engine_id] - + (q->properties.sdma_queue_id & 1) - * KFD_QUEUE_DOORBELL_MIRROR_OFFSET - + (q->properties.sdma_queue_id >> 1); + uint32_t *idx_offset = dev->shared_resources.sdma_doorbell_idx; + uint32_t valid_id = idx_offset[q->properties.sdma_engine_id] + + (q->properties.sdma_queue_id & 1) + * KFD_QUEUE_DOORBELL_MIRROR_OFFSET + + (q->properties.sdma_queue_id >> 1); + + if (restore_id && *restore_id != valid_id) + return -EINVAL; + q->doorbell_id = valid_id; } else { - /* For CP queues on SOC15 reserve a free doorbell ID */ - unsigned int found; - - found = find_first_zero_bit(qpd->doorbell_bitmap, - KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); - if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { - pr_debug("No doorbells available"); - return -EBUSY; + /* For CP queues on SOC15 */ + if (restore_id) { + /* make sure that ID is free */ + if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) { + return -EINVAL; + } + + q->doorbell_id = *restore_id; + } else { + /* or reserve a free doorbell ID */ + unsigned int found; + + found = find_first_zero_bit(qpd->doorbell_bitmap, + KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); + if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { + pr_debug("No doorbells available"); + return -EBUSY; + } + set_bit(found, qpd->doorbell_bitmap); + q->doorbell_id = found; } - set_bit(found, qpd->doorbell_bitmap); - q->doorbell_id = found; }
q->properties.doorbell_off = @@ -343,7 +363,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm, dqm->asic_ops.init_sdma_vm(dqm, q, qpd); }
- retval = allocate_doorbell(qpd, q); + retval = allocate_doorbell(qpd, q, qrd ? &qrd->doorbell_id : NULL); if (retval) goto out_deallocate_hqd;
@@ -1314,7 +1334,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, goto out; }
- retval = allocate_doorbell(qpd, q); + retval = allocate_doorbell(qpd, q, qrd ? &qrd->doorbell_id : NULL); if (retval) goto out_deallocate_sdma_queue;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index bd518340c38c..a4a0713ca9a3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -472,6 +472,7 @@ enum KFD_QUEUE_PRIORITY { struct queue_restore_data { uint32_t qid; uint32_t sdma_id; + uint32_t doorbell_id; };
struct queue_properties {