Re: [Patch v4 13/24] drm/amdkfd: CRIU checkpoint and restore queue mqds

10 Jan 2022

On 2021-12-22 7:37 p.m., Rajneesh Bhardwaj wrote:
...
From: David Yat Sin david.yatsin@amd.com
Checkpoint contents of queue MQD's on CRIU dump and restore them during
CRIU restore.
Signed-off-by: David Yat Sin david.yatsin@amd.com
David has an update for this patch to fix up the doorbell offset in the 
restored SDMA MQD.
Regards,
   Felix
...

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c      |   2 +-
  drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c       |   2 +-
  .../drm/amd/amdkfd/kfd_device_queue_manager.c |  72 +++++++-
  .../drm/amd/amdkfd/kfd_device_queue_manager.h |  14 +-
  drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h  |   7 +
  .../gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c  |  67 ++++++++
  .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c  |  68 ++++++++
  .../gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c   |  68 ++++++++
  .../gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c   |  69 ++++++++
  drivers/gpu/drm/amd/amdkfd/kfd_priv.h         |   5 +
  .../amd/amdkfd/kfd_process_queue_manager.c    | 158 ++++++++++++++++--
  11 files changed, 506 insertions(+), 26 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 3fb155f756fd..146879cd3f2b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -312,7 +312,7 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
   		p->pasid,
   		dev->id);

err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id, NULL,


err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id, NULL, NULL,
	&doorbell_offset_in_process);
if (err != 0)
goto err_create_queue;

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
index 0c50e67e2b51..3a5303ebcabf 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
@@ -185,7 +185,7 @@ static int dbgdev_register_diq(struct kfd_dbgdev *dbgdev)
   properties.type = KFD_QUEUE_TYPE_DIQ;
status = pqm_create_queue(dbgdev->pqm, dbgdev->dev, NULL,

		&properties, &qid, NULL, NULL);




		&properties, &qid, NULL, NULL, NULL);


if (status) {
pr_err("Failed to create DIQ\n");


diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index a0f5b8533a03..a92274f9f1f7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -331,7 +331,8 @@ static void deallocate_vmid(struct device_queue_manager *dqm,
  static int create_queue_nocpsch(struct device_queue_manager *dqm,
   			struct queue *q,
   			struct qcm_process_device *qpd,

		const struct kfd_criu_queue_priv_data *qd)




		const struct kfd_criu_queue_priv_data *qd,


		const void *restore_mqd)

{
struct mqd_manager *mqd_mgr;
int retval;

@@ -390,8 +391,14 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
   	retval = -ENOMEM;
   	goto out_deallocate_doorbell;
   }

mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
		&q->gart_mqd_addr, &q->properties);





if (qd)
mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,


		     &q->properties, restore_mqd);


else
mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,


			&q->gart_mqd_addr, &q->properties);


if (q->properties.is_active) {
  if (!dqm->sched_running) {
  	WARN_ONCE(1, "Load non-HWS mqd while stopped\n");

@@ -1339,7 +1346,8 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
   		struct qcm_process_device *qpd,

	const struct kfd_criu_queue_priv_data *qd)




	const struct kfd_criu_queue_priv_data *qd,


	const void *restore_mqd)

{
int retval;
struct mqd_manager *mqd_mgr;

@@ -1385,8 +1393,12 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
    * updates the is_evicted flag but is a no-op otherwise.
    */
   q->properties.is_evicted = !!qpd->evicted;

mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
		&q->gart_mqd_addr, &q->properties);




if (qd)

mqd_mgr->restore_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj, &q->gart_mqd_addr,


		     &q->properties, restore_mqd);


else

mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,


			&q->gart_mqd_addr, &q->properties);


list_add(&q->list, &qpd->queues_list);
qpd->queue_count++;


@@ -1774,6 +1786,50 @@ static int get_wave_state(struct device_queue_manager *dqm,
   		ctl_stack_used_size, save_area_used_size);
  }
+static void get_queue_checkpoint_info(struct device_queue_manager *dqm,

	const struct queue *q,


	u32 *mqd_size)



+{

struct mqd_manager *mqd_mgr;
enum KFD_MQD_TYPE mqd_type =
	get_mqd_type_from_queue_type(q->properties.type);



dqm_lock(dqm);
mqd_mgr = dqm->mqd_mgrs[mqd_type];
*mqd_size = mqd_mgr->mqd_size;

dqm_unlock(dqm);

+}



+static int checkpoint_mqd(struct device_queue_manager *dqm,

	  const struct queue *q,


	  void *mqd)



+{

struct mqd_manager *mqd_mgr;
int r = 0;
enum KFD_MQD_TYPE mqd_type =
	get_mqd_type_from_queue_type(q->properties.type);



dqm_lock(dqm);

if (q->properties.is_active || !q->device->cwsr_enabled) {
r = -EINVAL;


goto dqm_unlock;


}

mqd_mgr = dqm->mqd_mgrs[mqd_type];
if (!mqd_mgr->checkpoint_mqd) {
r = -EOPNOTSUPP;


goto dqm_unlock;


}

mqd_mgr->checkpoint_mqd(mqd_mgr, q->mqd, mqd);


+dqm_unlock:

dqm_unlock(dqm);
return r;

+}

static int process_termination_cpsch(struct device_queue_manager *dqm,
  struct qcm_process_device *qpd)
{

@@ -1950,6 +2006,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
   	dqm->ops.evict_process_queues = evict_process_queues_cpsch;
   	dqm->ops.restore_process_queues = restore_process_queues_cpsch;
   	dqm->ops.get_wave_state = get_wave_state;

dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;


dqm->ops.checkpoint_mqd = checkpoint_mqd;

break;
case KFD_SCHED_POLICY_NO_HWS:
/* initialize dqm for no cp scheduling */

@@ -1969,6 +2027,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
   	dqm->ops.restore_process_queues =
   		restore_process_queues_nocpsch;
   	dqm->ops.get_wave_state = get_wave_state;

dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;


dqm->ops.checkpoint_mqd = checkpoint_mqd;

break;
default:
pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 6fa43215e3e2..ebd7d4d3772b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -81,13 +81,18 @@ struct device_process_node {


@get_wave_state: Retrieves context save state and optionally copies the
control stack, if kept in the MQD, to the given userspace address.







@get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint.







@checkpoint_mqd: checkpoint queue MQD contents for CRIU.

*/
struct device_queue_manager_ops {
 int	(*create_queue)(struct device_queue_manager *dqm,
  		struct queue *q,
  		struct qcm_process_device *qpd,



		const struct kfd_criu_queue_priv_data *qd);




		const struct kfd_criu_queue_priv_data *qd,


		const void *restore_mqd);


int	(*destroy_queue)(struct device_queue_manager *dqm,
		struct qcm_process_device *qpd,


@@ -135,6 +140,13 @@ struct device_queue_manager_ops {
   			  void __user *ctl_stack,
   			  u32 *ctl_stack_used_size,
   			  u32 *save_area_used_size);


void	(*get_queue_checkpoint_info)(struct device_queue_manager *dqm,

		  const struct queue *q, u32 *mqd_size);



int	(*checkpoint_mqd)(struct device_queue_manager *dqm,

		  const struct queue *q,


		  void *mqd);

};
struct device_queue_manager_asic_ops {


diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 965e17c5dbb4..cebb2877a505 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -100,6 +100,13 @@ struct mqd_manager {
   			  u32 *ctl_stack_used_size,
   			  u32 *save_area_used_size);

void	(*checkpoint_mqd)(struct mqd_manager *mm, void *mqd, void *mqd_dst);

void	(*restore_mqd)(struct mqd_manager *mm, void **mqd,
		struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,


		struct queue_properties *p,


		const void *mqd_src);


#if defined(CONFIG_DEBUG_FS)
 int	(*debugfs_show_mqd)(struct seq_file *m, void *data);
#endif

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index e9a8e21e144e..dee90e72f672 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -280,6 +280,69 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
   				pipe_id, queue_id);
  }
+static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+{

struct cik_mqd *m;

m = get_mqd(mqd);

memcpy(mqd_dst, m, sizeof(struct cik_mqd));

+}



+static void restore_mqd(struct mqd_manager *mm, void **mqd,

	struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,


	struct queue_properties *qp,


	const void *mqd_src)



+{

uint64_t addr;
struct cik_mqd *m;

m = (struct cik_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;

memcpy(m, mqd_src, sizeof(*m));

*mqd = m;
if (gart_addr)
*gart_addr = addr;



m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(qp->doorbell_off);

pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
	m->cp_hqd_pq_doorbell_control);



qp->is_active = 0;

+}



+static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+{

struct cik_sdma_rlc_registers *m;

m = get_sdma_mqd(mqd);

memcpy(mqd_dst, m, sizeof(struct cik_sdma_rlc_registers));

+}



+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,

		struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,


		struct queue_properties *qp,


		const void *mqd_src)



+{

uint64_t addr;
struct cik_sdma_rlc_registers *m;

m = (struct cik_sdma_rlc_registers *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;

memcpy(m, mqd_src, sizeof(*m));

*mqd = m;
if (gart_addr)
*gart_addr = addr;



qp->is_active = 0;

+}

/*
preempt type here is ignored because there is only one way
to preempt sdma queue



@@ -394,6 +457,8 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
   	mqd->update_mqd = update_mqd;
   	mqd->destroy_mqd = destroy_mqd;
   	mqd->is_occupied = is_occupied;

mqd->checkpoint_mqd = checkpoint_mqd;


mqd->restore_mqd = restore_mqd;

mqd->mqd_size = sizeof(struct cik_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;

@@ -434,6 +499,8 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
   	mqd->update_mqd = update_mqd_sdma;
   	mqd->destroy_mqd = destroy_mqd_sdma;
   	mqd->is_occupied = is_occupied_sdma;

mqd->checkpoint_mqd = checkpoint_mqd_sdma;


mqd->restore_mqd = restore_mqd_sdma;

mqd->mqd_size = sizeof(struct cik_sdma_rlc_registers);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index d74d8a6ac27a..36109547494e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -285,6 +285,41 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
   return 0;
  }
+static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+{

struct v10_compute_mqd *m;

m = get_mqd(mqd);

memcpy(mqd_dst, m, sizeof(struct v10_compute_mqd));

+}



+static void restore_mqd(struct mqd_manager *mm, void **mqd,

	struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,


	struct queue_properties *qp,


	const void *mqd_src)



+{

uint64_t addr;
struct v10_compute_mqd *m;

m = (struct v10_compute_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;

memcpy(m, mqd_src, sizeof(*m));

*mqd = m;
if (gart_addr)
*gart_addr = addr;



m->cp_hqd_pq_doorbell_control =
qp->doorbell_off <<


	CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;


pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
	m->cp_hqd_pq_doorbell_control);



qp->is_active = 0;

+}

static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
  	struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
  	struct queue_properties *q)

@@ -373,6 +408,35 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
   return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
  }
+static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+{

struct v10_sdma_mqd *m;

m = get_sdma_mqd(mqd);

memcpy(mqd_dst, m, sizeof(struct v10_sdma_mqd));

+}



+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,

	     struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,


	     struct queue_properties *qp,


	     const void *mqd_src)



+{

uint64_t addr;
struct v10_sdma_mqd *m;

m = (struct v10_sdma_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;

memcpy(m, mqd_src, sizeof(*m));

*mqd = m;
if (gart_addr)
*gart_addr = addr;



qp->is_active = 0;

+}

#if defined(CONFIG_DEBUG_FS)
static int debugfs_show_mqd(struct seq_file *m, void *data)


@@ -417,6 +481,8 @@ struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
   	mqd->is_occupied = is_occupied;
   	mqd->mqd_size = sizeof(struct v10_compute_mqd);
   	mqd->get_wave_state = get_wave_state;

mqd->checkpoint_mqd = checkpoint_mqd;


mqd->restore_mqd = restore_mqd;

#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;
#endif

@@ -460,6 +526,8 @@ struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
   	mqd->update_mqd = update_mqd_sdma;
   	mqd->destroy_mqd = destroy_mqd_sdma;
   	mqd->is_occupied = is_occupied_sdma;

mqd->checkpoint_mqd = checkpoint_mqd_sdma;


mqd->restore_mqd = restore_mqd_sdma;

mqd->mqd_size = sizeof(struct v10_sdma_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 326eb2285029..86ded61fbdeb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -340,6 +340,41 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
   return 0;
  }
+static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+{

struct v9_mqd *m;

m = get_mqd(mqd);

memcpy(mqd_dst, m, sizeof(struct v9_mqd));

+}



+static void restore_mqd(struct mqd_manager *mm, void **mqd,

	struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,


	struct queue_properties *qp,


	const void *mqd_src)



+{

uint64_t addr;
struct v9_mqd *m;

m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;

memcpy(m, mqd_src, sizeof(*m));

*mqd = m;
if (gart_addr)
*gart_addr = addr;



m->cp_hqd_pq_doorbell_control =
qp->doorbell_off <<


	CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;


pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
		m->cp_hqd_pq_doorbell_control);



qp->is_active = 0;

+}

static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
  	struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
  	struct queue_properties *q)

@@ -428,6 +463,35 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
   return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
  }
+static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+{

struct v9_sdma_mqd *m;

m = get_sdma_mqd(mqd);

memcpy(mqd_dst, m, sizeof(struct v9_sdma_mqd));

+}



+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,

	     struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,


	     struct queue_properties *qp,


	     const void *mqd_src)



+{

uint64_t addr;
struct v9_sdma_mqd *m;

m = (struct v9_sdma_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;

memcpy(m, mqd_src, sizeof(*m));

*mqd = m;
if (gart_addr)
*gart_addr = addr;



qp->is_active = 0;

+}

#if defined(CONFIG_DEBUG_FS)
static int debugfs_show_mqd(struct seq_file *m, void *data)


@@ -470,6 +534,8 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
   	mqd->destroy_mqd = destroy_mqd;
   	mqd->is_occupied = is_occupied;
   	mqd->get_wave_state = get_wave_state;

mqd->checkpoint_mqd = checkpoint_mqd;


mqd->restore_mqd = restore_mqd;

mqd->mqd_size = sizeof(struct v9_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;

@@ -510,6 +576,8 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
   	mqd->update_mqd = update_mqd_sdma;
   	mqd->destroy_mqd = destroy_mqd_sdma;
   	mqd->is_occupied = is_occupied_sdma;

mqd->checkpoint_mqd = checkpoint_mqd_sdma;


mqd->restore_mqd = restore_mqd_sdma;

mqd->mqd_size = sizeof(struct v9_sdma_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index d456e950ce1d..e0204392c6dc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -306,6 +306,42 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
   return 0;
  }
+static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+{

struct vi_mqd *m;

m = get_mqd(mqd);

memcpy(mqd_dst, m, sizeof(struct vi_mqd));

+}



+static void restore_mqd(struct mqd_manager *mm, void **mqd,

	struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,


	struct queue_properties *qp,


	const void *mqd_src)



+{

uint64_t addr;
struct vi_mqd *m;

m = (struct vi_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;

memcpy(m, mqd_src, sizeof(*m));

*mqd = m;
if (gart_addr)
*gart_addr = addr;



m->cp_hqd_pq_doorbell_control =
qp->doorbell_off <<


	CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;


pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
	m->cp_hqd_pq_doorbell_control);



qp->is_active = 0;

+}


static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
  	struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
  	struct queue_properties *q)

@@ -399,6 +435,35 @@ static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
   return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
  }
+static void checkpoint_mqd_sdma(struct mqd_manager *mm, void *mqd, void *mqd_dst)
+{

struct vi_sdma_mqd *m;

m = get_sdma_mqd(mqd);

memcpy(mqd_dst, m, sizeof(struct vi_sdma_mqd));

+}



+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,

	     struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,


	     struct queue_properties *qp,


	     const void *mqd_src)



+{

uint64_t addr;
struct vi_sdma_mqd *m;

m = (struct vi_sdma_mqd *) mqd_mem_obj->cpu_ptr;
addr = mqd_mem_obj->gpu_addr;

memcpy(m, mqd_src, sizeof(*m));

*mqd = m;
if (gart_addr)
*gart_addr = addr;



qp->is_active = 0;

+}

#if defined(CONFIG_DEBUG_FS)
static int debugfs_show_mqd(struct seq_file *m, void *data)


@@ -441,6 +506,8 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
   	mqd->destroy_mqd = destroy_mqd;
   	mqd->is_occupied = is_occupied;
   	mqd->get_wave_state = get_wave_state;

mqd->checkpoint_mqd = checkpoint_mqd;


mqd->restore_mqd = restore_mqd;

mqd->mqd_size = sizeof(struct vi_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd;

@@ -481,6 +548,8 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
   	mqd->update_mqd = update_mqd_sdma;
   	mqd->destroy_mqd = destroy_mqd_sdma;
   	mqd->is_occupied = is_occupied_sdma;

mqd->checkpoint_mqd = checkpoint_mqd_sdma;


mqd->restore_mqd = restore_mqd_sdma;

mqd->mqd_size = sizeof(struct vi_sdma_mqd);
#if defined(CONFIG_DEBUG_FS)
mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 8272bd5c4600..9a381494eb67 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -1158,6 +1158,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
   		    struct queue_properties *properties,
   		    unsigned int *qid,
   		    const struct kfd_criu_queue_priv_data *q_data,

	    const void *restore_mqd,
    uint32_t *p_doorbell_offset_in_process);

int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
int pqm_update_queue_properties(struct process_queue_manager *pqm, unsigned int qid,

@@ -1180,6 +1181,10 @@ int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
   		      uint64_t fence_value,
   		      unsigned int timeout_ms);
+int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,

		  unsigned int qid,


		  u32 *mqd_size);


/* Packet Manager */
#define KFD_FENCE_COMPLETED (100)


diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 25cf97d97d87..97e794d6bb76 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -209,6 +209,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
   		    struct queue_properties *properties,
   		    unsigned int *qid,
   		    const struct kfd_criu_queue_priv_data *q_data,

	    const void *restore_mqd,
    uint32_t *p_doorbell_offset_in_process)

{
int retval;

@@ -273,7 +274,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
   		goto err_create_queue;
   	pqn->q = q;
   	pqn->kq = NULL;

retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data);




retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data, restore_mqd);

print_queue(q);
break;

@@ -293,7 +294,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
   		goto err_create_queue;
   	pqn->q = q;
   	pqn->kq = NULL;

retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data);




retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data, restore_mqd);

print_queue(q);
break;
case KFD_QUEUE_TYPE_DIQ:

@@ -518,12 +519,26 @@ int pqm_get_wave_state(struct process_queue_manager *pqm,
   					       save_area_used_size);
  }



+static int get_queue_data_sizes(struct kfd_process_device *pdd, struct queue *q, uint32_t *mqd_size)
+{

int ret;

ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm, q->properties.queue_id, mqd_size);
if (ret)
pr_err("Failed to get queue dump info (%d)\n", ret);



return ret;

+}

int kfd_process_get_queue_info(struct kfd_process *p,
  	       uint32_t *num_queues,
  	       uint64_t *priv_data_sizes)
{

uint32_t extra_data_sizes = 0;
struct queue *q;
int i;

int ret;
*num_queues = 0;


@@ -535,23 +550,53 @@ int kfd_process_get_queue_info(struct kfd_process *p,
   		if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
   			q->properties.type == KFD_QUEUE_TYPE_SDMA ||
   			q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {




		uint32_t mqd_size;
	*num_queues = *num_queues + 1;



		ret = get_queue_data_sizes(pdd, q, &mqd_size);


		if (ret)


			return ret;



		extra_data_sizes += mqd_size;
} else {
	pr_err("Unsupported queue type (%d)\n", q->properties.type);
	return -EOPNOTSUPP;
}

}
}


*priv_data_sizes = *num_queues * sizeof(struct kfd_criu_queue_priv_data);


*priv_data_sizes = extra_data_sizes +

		(*num_queues * sizeof(struct kfd_criu_queue_priv_data));


return 0;
}


-static void criu_checkpoint_queue(struct kfd_process_device *pdd,
+static int pqm_checkpoint_mqd(struct process_queue_manager *pqm, unsigned int qid, void *mqd)
+{

struct process_queue_node *pqn;

pqn = get_queue_by_qid(pqm, qid);
if (!pqn) {
pr_debug("amdkfd: No queue %d exists for operation\n", qid);


return -EFAULT;


}

if (!pqn->q->device->dqm->ops.checkpoint_mqd) {
pr_err("amdkfd: queue dumping not supported on this device\n");


return -EOPNOTSUPP;


}

return pqn->q->device->dqm->ops.checkpoint_mqd(pqn->q->device->dqm, pqn->q, mqd);

+}



+static int criu_checkpoint_queue(struct kfd_process_device *pdd,
   		   struct queue *q,
   		   struct kfd_criu_queue_priv_data *q_data)
  {

uint8_t *mqd;
int ret;

mqd = (void *)(q_data + 1);
q_data->gpu_id = pdd->dev->id;
 q_data->type = q->properties.type;
 q_data->format = q->properties.format;

@@ -577,7 +622,14 @@ static void criu_checkpoint_queue(struct kfd_process_device *pdd,
   q_data->ctx_save_restore_area_size =
   	q->properties.ctx_save_restore_area_size;

ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd);

if (ret) {

pr_err("Failed checkpoint queue_mqd (%d)\n", ret);


return ret;


}

pr_debug("Dumping Queue: gpu_id:%x queue_id:%u\n", q_data->gpu_id, q_data->q_id);

return ret;
}
static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,


@@ -585,15 +637,16 @@ static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,
   			   unsigned int *q_index,
   			   uint64_t *queues_priv_data_offset)
  {

struct kfd_criu_queue_priv_data *q_data;


unsigned int q_private_data_size = 0;
uint8_t *q_private_data = NULL; /* Local buffer to store individual queue private data */
struct queue *q;
int ret = 0;


q_data = kzalloc(sizeof(*q_data), GFP_KERNEL);
if (!q_data)
return -ENOMEM;


list_for_each_entry(q, &pdd->qpd.queues_list, list) {


struct kfd_criu_queue_priv_data *q_data;


uint64_t q_data_size;


uint32_t mqd_size;


if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE &&
  	q->properties.type != KFD_QUEUE_TYPE_SDMA &&
  	q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI) {

@@ -602,19 +655,46 @@ static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,
   		return -EOPNOTSUPP;
   	}

criu_checkpoint_queue(pdd, q, q_data);




ret = get_queue_data_sizes(pdd, q, &mqd_size);


if (ret)


	break;



q_data_size = sizeof(*q_data) + mqd_size;



/* Increase local buffer space if needed */


if (q_private_data_size < q_data_size) {


	kfree(q_private_data);



	q_private_data = kzalloc(q_data_size, GFP_KERNEL);


	if (!q_private_data) {


		ret = -ENOMEM;


		break;


	}


	q_private_data_size = q_data_size;


}



q_data = (struct kfd_criu_queue_priv_data *)q_private_data;



/* data stored in this order: priv_data, mqd */


q_data->mqd_size = mqd_size;



ret = criu_checkpoint_queue(pdd, q, q_data);


if (ret)


	break;


q_data->object_type = KFD_CRIU_OBJECT_TYPE_QUEUE;


ret = copy_to_user(user_priv + *queues_priv_data_offset, q_data, sizeof(*q_data));




ret = copy_to_user(user_priv + *queues_priv_data_offset,


		q_data, q_data_size);

if (ret) {
	ret = -EFAULT;
	break;
}


*queues_priv_data_offset += sizeof(*q_data);




*queues_priv_data_offset += q_data_size;

*q_index = *q_index + 1;
}


kfree(q_data);


kfree(q_private_data);
return ret;
}


@@ -671,11 +751,12 @@ int kfd_criu_restore_queue(struct kfd_process *p,
   		   uint64_t max_priv_data_size)
  {
   struct kfd_criu_queue_priv_data *q_data;

uint8_t *mqd, *q_extra_data = NULL;
struct kfd_process_device *pdd;


struct kfd_dev *dev;


uint64_t q_extra_data_size;
struct queue_properties qp;
unsigned int queue_id;





struct kfd_dev *dev;
int ret = 0;
if (*priv_data_offset + sizeof(*q_data) > max_priv_data_size)


@@ -692,6 +773,26 @@ int kfd_criu_restore_queue(struct kfd_process *p,
   }
*priv_data_offset += sizeof(*q_data);

q_extra_data_size = q_data->mqd_size;


if (*priv_data_offset + q_extra_data_size > max_priv_data_size) {

ret = -EINVAL;


goto exit;


}


q_extra_data = kmalloc(q_extra_data_size, GFP_KERNEL);

if (!q_extra_data) {

ret = -ENOMEM;


goto exit;


}


ret = copy_from_user(q_extra_data, user_priv_ptr + *priv_data_offset, q_extra_data_size);

if (ret) {

ret = -EFAULT;


goto exit;


}


*priv_data_offset += q_extra_data_size;
dev = kfd_device_by_id(q_data->gpu_id);
if (!dev) {


@@ -708,6 +809,8 @@ int kfd_criu_restore_queue(struct kfd_process *p,
   	ret = -EFAULT;
   	return ret;
   }

/* data stored in this order: mqd */

mqd = q_extra_data;
memset(&qp, 0, sizeof(qp));
ret = set_queue_properties_from_criu(&qp, q_data);


@@ -716,7 +819,7 @@ int kfd_criu_restore_queue(struct kfd_process *p,
print_queue_properties(&qp);

ret = pqm_create_queue(&p->pqm, pdd->dev, NULL, &qp, &queue_id, q_data, NULL);


ret = pqm_create_queue(&p->pqm, pdd->dev, NULL, &qp, &queue_id, q_data, mqd, NULL);
if (ret) {
pr_err("Failed to create new queue err:%d\n", ret);
ret = -EINVAL;

@@ -733,6 +836,27 @@ int kfd_criu_restore_queue(struct kfd_process *p,
   return ret;
  }
+int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,

		  unsigned int qid,


		  uint32_t *mqd_size)



+{

struct process_queue_node *pqn;

pqn = get_queue_by_qid(pqm, qid);
if (!pqn) {
pr_debug("amdkfd: No queue %d exists for operation\n", qid);


return -EFAULT;


}

if (!pqn->q->device->dqm->ops.get_queue_checkpoint_info) {
pr_err("amdkfd: queue dumping not supported on this device\n");


return -EOPNOTSUPP;


}

pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm, pqn->q, mqd_size);
return 0;

+}

#if defined(CONFIG_DEBUG_FS)
int pqm_debugfs_mqds(struct seq_file *m, void *data)



    

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

Re: [Patch v4 13/24] drm/amdkfd: CRIU checkpoint and restore queue mqds