[RFC PATCH 13/17] drm/amdkfd: CRIU dump and restore queue mqds

1 May 2021

From: David Yat Sin david.yatsin@amd.com
Dump contents of queue MQD's on CRIU dump and restore them during CRIU
restore.
Signed-off-by: David Yat Sin david.yatsin@amd.com
Change-Id: If54f892fb6cb8a5bde8a993891dad0dbb997c239
---
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      | 35 ++++++++--
 .../drm/amd/amdkfd/kfd_device_queue_manager.c | 64 +++++++++++++++--
 .../drm/amd/amdkfd/kfd_device_queue_manager.h |  8 +++
 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h  |  8 +++
 .../gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c  | 66 ++++++++++++++++++
 .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c  | 67 ++++++++++++++++++
 .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c   | 68 +++++++++++++++++++
 .../gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c   | 67 ++++++++++++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h         |  8 +++
 .../amd/amdkfd/kfd_process_queue_manager.c    | 42 ++++++++++++
 10 files changed, 422 insertions(+), 11 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 1d2c2d986a44..b7f3aa759c17 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -1893,11 +1893,18 @@ static int kfd_devinfo_restore(struct kfd_process *p, struct kfd_criu_devinfo_bu
static int get_queue_data_sizes(struct kfd_process_device *pdd,
    			struct queue *q,
-				uint32_t *cu_mask_size)
+				uint32_t *cu_mask_size,
+				uint32_t *mqd_size)
 {
    int ret = 0;
*cu_mask_size = sizeof(uint32_t) * (q->properties.cu_mask_count / 32);
+
+	ret = pqm_get_queue_dump_info(&pdd->process->pqm, q->properties.queue_id, mqd_size);
+	if (ret) {
+		pr_err("Failed to get queue dump info (%d)\n", ret);
+		return ret;
+	}
    return ret;
 }
@@ -1906,6 +1913,8 @@ static int criu_dump_queue(struct kfd_process_device *pdd,
                            struct kfd_criu_q_bucket *q_bucket,
    		   struct queue_restore_data *qrd)
 {
+	int ret = 0;
+
    q_bucket->gpu_id = pdd->dev->id;
    q_bucket->type = q->properties.type;
    q_bucket->format = q->properties.format;
@@ -1935,8 +1944,15 @@ static int criu_dump_queue(struct kfd_process_device *pdd,
    if (qrd->cu_mask_size)
    	memcpy(qrd->cu_mask, q->properties.cu_mask, qrd->cu_mask_size);
+	ret = pqm_dump_mqd(&pdd->process->pqm, q->properties.queue_id, qrd);
+	if (ret) {
+		pr_err("Failed dump queue_mqd (%d)\n", ret);
+		return ret;
+	}
+
    q_bucket->cu_mask_size = qrd->cu_mask_size;
-	return 0;
+	q_bucket->mqd_size = qrd->mqd_size;
+	return ret;
 }
static int criu_dump_queues_device(struct kfd_process_device *pdd,
@@ -1975,14 +1991,14 @@ static int criu_dump_queues_device(struct kfd_process_device *pdd,
    	memset(&q_bucket, 0, sizeof(q_bucket));
    	memset(&qrd, 0, sizeof(qrd));
-		ret = get_queue_data_sizes(pdd, q, &qrd.cu_mask_size);
+		ret = get_queue_data_sizes(pdd, q, &qrd.cu_mask_size, &qrd.mqd_size);
    	if (ret) {
    		pr_err("Failed to get queue dump info (%d)\n", ret);
    		ret = -EFAULT;
    		break;
    	}
-		q_data_size = qrd.cu_mask_size;
+		q_data_size = qrd.cu_mask_size + qrd.mqd_size;
/* Increase local buffer space if needed */
    	if (data_ptr_size < q_data_size) {
@@ -1998,6 +2014,7 @@ static int criu_dump_queues_device(struct kfd_process_device *pdd,
    	}
qrd.cu_mask = data_ptr;
+		qrd.mqd = data_ptr + qrd.cu_mask_size;
    	ret = criu_dump_queue(pdd, q, &q_bucket, &qrd);
    	if (ret)
    		break;
@@ -2317,7 +2334,7 @@ static int criu_restore_queues(struct kfd_process *p,
    		return ret;
    	}
-		q_data_size = q_bucket.cu_mask_size;
+		q_data_size = q_bucket.cu_mask_size + q_bucket.mqd_size;
/* Increase local buffer space if needed */
    	if (q_data_size > data_ptr_size) {
@@ -2343,6 +2360,9 @@ static int criu_restore_queues(struct kfd_process *p,
    	qrd.cu_mask_size = q_bucket.cu_mask_size;
    	qrd.cu_mask = data_ptr;
+		qrd.mqd_size = q_bucket.mqd_size;
+		qrd.mqd = data_ptr + qrd.cu_mask_size;
+
    	ret = criu_restore_queue(p, dev, pdd, &q_bucket, &qrd);
    	if (ret) {
    		pr_err("Failed to restore queue (%d)\n", ret);
@@ -2680,11 +2700,12 @@ static int kfd_ioctl_criu_helper(struct file *filep,
    			q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
u32 cu_mask_size = 0;
-				ret = get_queue_data_sizes(pdd, q, &cu_mask_size);
+				u32 mqd_size = 0;
+				ret = get_queue_data_sizes(pdd, q, &cu_mask_size, &mqd_size);
    			if (ret)
    				goto err_unlock;
-				queues_data_size += cu_mask_size;
+				queues_data_size += cu_mask_size + mqd_size;
    			q_index++;
    		} else {
    			pr_err("Unsupported queue type (%d)\n", q->properties.type);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 56250c0b5ca0..ee34229a41b0 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -376,8 +376,14 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
    	retval = -ENOMEM;
    	goto out_deallocate_doorbell;
    }
-	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
-				&q->gart_mqd_addr, &q->properties);
+
+	if (qrd)
+		mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
+				&q->gart_mqd_addr, &q->properties, qrd);
+	else
+		mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
+					&q->gart_mqd_addr, &q->properties);
+
    if (q->properties.is_active) {
    	if (!dqm->sched_running) {
    		WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
@@ -1359,8 +1365,13 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
     * updates the is_evicted flag but is a no-op otherwise.
     */
    q->properties.is_evicted = !!qpd->evicted;
-	mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
-				&q->gart_mqd_addr, &q->properties);
+
+	if (qrd)
+		mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
+				&q->gart_mqd_addr, &q->properties, qrd);
+	else
+		mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
+					&q->gart_mqd_addr, &q->properties);
list_add(&q->list, &qpd->queues_list);
    qpd->queue_count++;
@@ -1743,6 +1754,47 @@ static int get_wave_state(struct device_queue_manager *dqm,
    return r;
 }
+static void get_queue_dump_info(struct device_queue_manager *dqm,
+			const struct queue *q,
+			u32 *mqd_size)
+{
+	struct mqd_manager *mqd_mgr;
+	enum KFD_MQD_TYPE mqd_type =
+			get_mqd_type_from_queue_type(q->properties.type);
+
+	mqd_mgr = dqm->mqd_mgrs[mqd_type];
+	*mqd_size = mqd_mgr->mqd_size;
+}
+
+static int dump_mqd(struct device_queue_manager *dqm,
+			  const struct queue *q,
+			  struct queue_restore_data *qrd)
+{
+	struct mqd_manager *mqd_mgr;
+	int r = 0;
+	enum KFD_MQD_TYPE mqd_type =
+			get_mqd_type_from_queue_type(q->properties.type);
+
+	dqm_lock(dqm);
+
+	if (q->properties.is_active || !q->device->cwsr_enabled) {
+		r = -EINVAL;
+		goto dqm_unlock;
+	}
+
+	mqd_mgr = dqm->mqd_mgrs[mqd_type];
+	if (!mqd_mgr->dump_mqd) {
+		r = -ENOTSUPP;
+		goto dqm_unlock;
+	}
+
+	mqd_mgr->dump_mqd(mqd_mgr, q->mqd, qrd);
+
+dqm_unlock:
+	dqm_unlock(dqm);
+	return r;
+}
+
 static int process_termination_cpsch(struct device_queue_manager *dqm,
    	struct qcm_process_device *qpd)
 {
@@ -1917,6 +1969,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
    	dqm->ops.evict_process_queues = evict_process_queues_cpsch;
    	dqm->ops.restore_process_queues = restore_process_queues_cpsch;
    	dqm->ops.get_wave_state = get_wave_state;
+		dqm->ops.get_queue_dump_info = get_queue_dump_info;
+		dqm->ops.dump_mqd = dump_mqd;
    	break;
    case KFD_SCHED_POLICY_NO_HWS:
    	/* initialize dqm for no cp scheduling */
@@ -1936,6 +1990,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
    	dqm->ops.restore_process_queues =
    		restore_process_queues_nocpsch;
    	dqm->ops.get_wave_state = get_wave_state;
+		dqm->ops.get_queue_dump_info = get_queue_dump_info;
+		dqm->ops.dump_mqd = dump_mqd;
    	break;
    default:
    	pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index a5baf50fd6dc..27952e33a404 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -135,6 +135,14 @@ struct device_queue_manager_ops {
    			  void __user *ctl_stack,
    			  u32 *ctl_stack_used_size,
    			  u32 *save_area_used_size);
+
+	void	(*get_queue_dump_info)(struct device_queue_manager *dqm,
+				  const struct queue *q,
+				  u32 *mqd_size);
+
+	int	(*dump_mqd)(struct device_queue_manager *dqm,
+				  const struct queue *q,
+				  struct queue_restore_data *qrd);
 };
struct device_queue_manager_asic_ops {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index b5e2ea7550d4..82da00bc71aa 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -98,6 +98,14 @@ struct mqd_manager {
    			  u32 *ctl_stack_used_size,
    			  u32 *save_area_used_size);
+	void	(*dump_mqd)(struct mqd_manager *mm, void *mqd,
+				struct queue_restore_data *qrd);
+
+	void	(*restore_mqd)(struct mqd_manager *mm, void **mqd,
+				struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+				struct queue_properties *p,
+				const struct queue_restore_data *qrd);
+
 #if defined(CONFIG_DEBUG_FS)
    int	(*debugfs_show_mqd)(struct seq_file *m, void *data);
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index 064914e1e8d6..3f59466a5104 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -275,6 +275,68 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
    				pipe_id, queue_id);
 }
+static void dump_mqd(struct mqd_manager *mm, void *mqd,
+                       struct queue_restore_data *qrd)
+{
+	struct cik_mqd *m;
+	m = get_mqd(mqd);
+
+	memcpy(qrd->mqd, m, sizeof(struct cik_mqd));
+}
+
+static void restore_mqd(struct mqd_manager *mm, void **mqd,
+                       struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+                       struct queue_properties *qp,
+                       const struct queue_restore_data *qrd)
+{
+	uint64_t addr;
+	struct cik_mqd *m;
+	m = (struct cik_mqd *) mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memcpy(m, qrd->mqd, sizeof(*m));
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(qp->doorbell_off);
+
+	pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
+			m->cp_hqd_pq_doorbell_control);
+
+	qp->is_active = 0;
+}
+
+static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd,
+                       struct queue_restore_data *qrd)
+{
+	struct cik_sdma_rlc_registers *m;
+	m = get_sdma_mqd(mqd);
+
+	memcpy(qrd->mqd, m, sizeof(struct cik_sdma_rlc_registers));
+}
+
+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
+                       struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+                       struct queue_properties *qp,
+                       const struct queue_restore_data *qrd)
+{
+	uint64_t addr;
+	struct cik_sdma_rlc_registers *m;
+
+	m = (struct cik_sdma_rlc_registers *) mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memcpy(m, qrd->mqd, sizeof(*m));
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	qp->is_active = 0;
+}
+
 /*
  * preempt type here is ignored because there is only one way
  * to preempt sdma queue
@@ -388,6 +450,8 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
    	mqd->update_mqd = update_mqd;
    	mqd->destroy_mqd = destroy_mqd;
    	mqd->is_occupied = is_occupied;
+		mqd->dump_mqd = dump_mqd;
+		mqd->restore_mqd = restore_mqd;
    	mqd->mqd_size = sizeof(struct cik_mqd);
 #if defined(CONFIG_DEBUG_FS)
    	mqd->debugfs_show_mqd = debugfs_show_mqd;
@@ -428,6 +492,8 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
    	mqd->update_mqd = update_mqd_sdma;
    	mqd->destroy_mqd = destroy_mqd_sdma;
    	mqd->is_occupied = is_occupied_sdma;
+		mqd->dump_mqd = dump_mqd_sdma;
+		mqd->restore_mqd = restore_mqd_sdma;
    	mqd->mqd_size = sizeof(struct cik_sdma_rlc_registers);
 #if defined(CONFIG_DEBUG_FS)
    	mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index c7fb59ca597f..126c0dc63f10 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -283,6 +283,40 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
    return 0;
 }
+static void dump_mqd(struct mqd_manager *mm, void *mqd,
+                       struct queue_restore_data *qrd)
+{
+	struct v10_compute_mqd *m;
+	m = get_mqd(mqd);
+
+	memcpy(qrd->mqd, m, sizeof(struct v10_compute_mqd));
+}
+
+static void restore_mqd(struct mqd_manager *mm, void **mqd,
+                       struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+                       struct queue_properties *qp,
+                       const struct queue_restore_data *qrd)
+{
+	uint64_t addr;
+	struct v10_compute_mqd *m;
+	m = (struct v10_compute_mqd *) mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memcpy(m, qrd->mqd, sizeof(*m));
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	m->cp_hqd_pq_doorbell_control =
+		qp->doorbell_off <<
+			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+	pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
+			m->cp_hqd_pq_doorbell_control);
+
+	qp->is_active = 0;
+}
+
 static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
    		struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
    		struct queue_properties *q)
@@ -370,6 +404,35 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
    return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
 }
+static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd,
+                       struct queue_restore_data *qrd)
+{
+	struct v10_sdma_mqd *m;
+	m = get_sdma_mqd(mqd);
+
+	memcpy(qrd->mqd, m, sizeof(struct v10_sdma_mqd));
+}
+
+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
+                       struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+                       struct queue_properties *qp,
+                       const struct queue_restore_data *qrd)
+{
+	uint64_t addr;
+	struct v10_sdma_mqd *m;
+
+	m = (struct v10_sdma_mqd *) mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memcpy(m, qrd->mqd, sizeof(*m));
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	qp->is_active = 0;
+}
+
 #if defined(CONFIG_DEBUG_FS)
static int debugfs_show_mqd(struct seq_file *m, void *data)
@@ -414,6 +477,8 @@ struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
    	mqd->is_occupied = is_occupied;
    	mqd->mqd_size = sizeof(struct v10_compute_mqd);
    	mqd->get_wave_state = get_wave_state;
+		mqd->dump_mqd = dump_mqd;
+		mqd->restore_mqd = restore_mqd;
 #if defined(CONFIG_DEBUG_FS)
    	mqd->debugfs_show_mqd = debugfs_show_mqd;
 #endif
@@ -458,6 +523,8 @@ struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
    	mqd->destroy_mqd = destroy_mqd_sdma;
    	mqd->is_occupied = is_occupied_sdma;
    	mqd->mqd_size = sizeof(struct v10_sdma_mqd);
+		mqd->dump_mqd = dump_mqd_sdma;
+		mqd->restore_mqd = restore_mqd_sdma;
 #if defined(CONFIG_DEBUG_FS)
    	mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 7f4e102ff4bd..8318192920e4 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -338,6 +338,41 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
    return 0;
 }
+static void dump_mqd(struct mqd_manager *mm, void *mqd,
+                       struct queue_restore_data *qrd)
+{
+	struct v9_mqd *m;
+	m = get_mqd(mqd);
+
+	memcpy(qrd->mqd, m, sizeof(struct v9_mqd));
+}
+
+static void restore_mqd(struct mqd_manager *mm, void **mqd,
+                       struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+                       struct queue_properties *qp,
+                       const struct queue_restore_data *qrd)
+{
+	uint64_t addr;
+	struct v9_mqd *m;
+	m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memcpy(m, qrd->mqd, sizeof(*m));
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	/* Control stack is located one page after MQD. */
+	m->cp_hqd_pq_doorbell_control =
+		qp->doorbell_off <<
+			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+	pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
+				m->cp_hqd_pq_doorbell_control);
+
+	qp->is_active = 0;
+}
+
 static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
    		struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
    		struct queue_properties *q)
@@ -425,6 +460,35 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
    return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
 }
+static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd,
+                       struct queue_restore_data *qrd)
+{
+	struct v9_sdma_mqd *m;
+	m = get_sdma_mqd(mqd);
+
+	memcpy(qrd->mqd, m, sizeof(struct v9_sdma_mqd));
+}
+
+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
+                       struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+                       struct queue_properties *qp,
+                       const struct queue_restore_data *qrd)
+{
+	uint64_t addr;
+	struct v9_sdma_mqd *m;
+
+	m = (struct v9_sdma_mqd *) mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memcpy(m, qrd->mqd, sizeof(*m));
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	qp->is_active = 0;
+}
+
 #if defined(CONFIG_DEBUG_FS)
static int debugfs_show_mqd(struct seq_file *m, void *data)
@@ -467,6 +531,8 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
    	mqd->destroy_mqd = destroy_mqd;
    	mqd->is_occupied = is_occupied;
    	mqd->get_wave_state = get_wave_state;
+		mqd->dump_mqd = dump_mqd;
+		mqd->restore_mqd = restore_mqd;
    	mqd->mqd_size = sizeof(struct v9_mqd);
 #if defined(CONFIG_DEBUG_FS)
    	mqd->debugfs_show_mqd = debugfs_show_mqd;
@@ -507,6 +573,8 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
    	mqd->update_mqd = update_mqd_sdma;
    	mqd->destroy_mqd = destroy_mqd_sdma;
    	mqd->is_occupied = is_occupied_sdma;
+		mqd->dump_mqd = dump_mqd_sdma;
+		mqd->restore_mqd = restore_mqd_sdma;
    	mqd->mqd_size = sizeof(struct v9_sdma_mqd);
 #if defined(CONFIG_DEBUG_FS)
    	mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 33dbd22d290f..ff436737b717 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -303,6 +303,40 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
    return 0;
 }
+static void dump_mqd(struct mqd_manager *mm, void *mqd,
+                       struct queue_restore_data *qrd)
+{
+	struct vi_mqd *m;
+	m = get_mqd(mqd);
+
+	memcpy(qrd->mqd, m, sizeof(struct vi_mqd));
+}
+
+static void restore_mqd(struct mqd_manager *mm, void **mqd,
+                       struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+                       struct queue_properties *qp,
+                       const struct queue_restore_data *qrd)
+{
+	uint64_t addr;
+	struct vi_mqd *m;
+	m = (struct vi_mqd *) mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memcpy(m, qrd->mqd, sizeof(*m));
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	m->cp_hqd_pq_doorbell_control =
+		qp->doorbell_off <<
+			CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+	pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
+			m->cp_hqd_pq_doorbell_control);
+
+	qp->is_active = 0;
+}
+
 static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
    		struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
    		struct queue_properties *q)
@@ -394,6 +428,35 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
    return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
 }
+static void dump_mqd_sdma(struct mqd_manager *mm, void *mqd,
+                       struct queue_restore_data *qrd)
+{
+	struct vi_sdma_mqd *m;
+	m = get_sdma_mqd(mqd);
+
+	memcpy(qrd->mqd, m, sizeof(struct vi_sdma_mqd));
+}
+
+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
+                       struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+                       struct queue_properties *qp,
+                       const struct queue_restore_data *qrd)
+{
+	uint64_t addr;
+	struct vi_sdma_mqd *m;
+
+	m = (struct vi_sdma_mqd *) mqd_mem_obj->cpu_ptr;
+	addr = mqd_mem_obj->gpu_addr;
+
+	memcpy(m, qrd->mqd, sizeof(*m));
+
+	*mqd = m;
+	if (gart_addr)
+		*gart_addr = addr;
+
+	qp->is_active = 0;
+}
+
 #if defined(CONFIG_DEBUG_FS)
static int debugfs_show_mqd(struct seq_file *m, void *data)
@@ -436,6 +499,8 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
    	mqd->destroy_mqd = destroy_mqd;
    	mqd->is_occupied = is_occupied;
    	mqd->get_wave_state = get_wave_state;
+		mqd->dump_mqd = dump_mqd;
+		mqd->restore_mqd = restore_mqd;
    	mqd->mqd_size = sizeof(struct vi_mqd);
 #if defined(CONFIG_DEBUG_FS)
    	mqd->debugfs_show_mqd = debugfs_show_mqd;
@@ -476,6 +541,8 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
    	mqd->update_mqd = update_mqd_sdma;
    	mqd->destroy_mqd = destroy_mqd_sdma;
    	mqd->is_occupied = is_occupied_sdma;
+		mqd->dump_mqd = dump_mqd_sdma;
+		mqd->restore_mqd = restore_mqd_sdma;
    	mqd->mqd_size = sizeof(struct vi_sdma_mqd);
 #if defined(CONFIG_DEBUG_FS)
    	mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 55180a6fe102..defea19c8bb9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -476,6 +476,8 @@ struct queue_restore_data {
void *cu_mask;
    uint32_t cu_mask_size;
+	void *mqd;
+	uint32_t mqd_size;
 };
struct queue_properties {
@@ -1087,6 +1089,12 @@ int pqm_get_wave_state(struct process_queue_manager *pqm,
 int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
    		      uint64_t fence_value,
    		      unsigned int timeout_ms);
+int pqm_get_queue_dump_info(struct process_queue_manager *pqm,
+			unsigned int qid,
+			u32 *mqd_size);
+int pqm_dump_mqd(struct process_queue_manager *pqm,
+		       unsigned int qid,
+		       struct queue_restore_data* qrd);
/* Packet Manager */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 0ca7db288b9f..8516e93b521b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -527,6 +527,48 @@ int pqm_get_wave_state(struct process_queue_manager *pqm,
    					       save_area_used_size);
 }
+int pqm_get_queue_dump_info(struct process_queue_manager *pqm,
+			unsigned int qid,
+			u32 *mqd_size)
+{
+	struct process_queue_node *pqn;
+
+	pqn = get_queue_by_qid(pqm, qid);
+	if (!pqn) {
+		pr_debug("amdkfd: No queue %d exists for operation\n", qid);
+		return -EFAULT;
+	}
+
+	if (!pqn->q->device->dqm->ops.get_queue_dump_info) {
+		pr_err("amdkfd: queue dumping not supported on this device\n");
+		return -ENOTSUPP;
+	}
+
+	pqn->q->device->dqm->ops.get_queue_dump_info(pqn->q->device->dqm,
+						       pqn->q, mqd_size);
+	return 0;
+}
+
+int pqm_dump_mqd(struct process_queue_manager *pqm,
+		       unsigned int qid, struct queue_restore_data* qrd)
+{
+	struct process_queue_node *pqn;
+
+	pqn = get_queue_by_qid(pqm, qid);
+	if (!pqn) {
+		pr_debug("amdkfd: No queue %d exists for operation\n", qid);
+		return -EFAULT;
+	}
+
+	if (!pqn->q->device->dqm->ops.dump_mqd) {
+		pr_err("amdkfd: queue dumping not supported on this device\n");
+		return -ENOTSUPP;
+	}
+
+	return pqn->q->device->dqm->ops.dump_mqd(pqn->q->device->dqm,
+						       pqn->q, qrd);
+}
+
 #if defined(CONFIG_DEBUG_FS)
int pqm_debugfs_mqds(struct seq_file *m, void *data)
-- 
2.17.1


    

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

[RFC PATCH 13/17] drm/amdkfd: CRIU dump and restore queue mqds