From: Alex Sierra alex.sierra@amd.com

Remove per_device_list from kfd_process and replace it with a kfd_process_device pointers array of MAX_GPU_INSTANCES size. This helps to manage the kfd_process_devices binded to a specific kfd_process. Also, functions used by kfd_chardev to iterate over the list were removed, since they are not valid anymore. Instead, it was replaced by a local loop iterating the array.

Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 116 ++++++++---------- drivers/gpu/drm/amd/amdkfd/kfd_iommu.c | 8 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 20 +-- drivers/gpu/drm/amd/amdkfd/kfd_process.c | 108 ++++++++-------- .../amd/amdkfd/kfd_process_queue_manager.c | 6 +- 5 files changed, 111 insertions(+), 147 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 8cc51cec988a..8c87afce12df 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -874,52 +874,47 @@ static int kfd_ioctl_get_process_apertures(struct file *filp, { struct kfd_ioctl_get_process_apertures_args *args = data; struct kfd_process_device_apertures *pAperture; - struct kfd_process_device *pdd; + int i;

dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);

args->num_of_nodes = 0;

mutex_lock(&p->mutex); + /* Run over all pdd of the process */ + for (i = 0; i < p->n_pdds; i++) { + struct kfd_process_device *pdd = p->pdds[i]; + + pAperture = + &args->process_apertures[args->num_of_nodes]; + pAperture->gpu_id = pdd->dev->id; + pAperture->lds_base = pdd->lds_base; + pAperture->lds_limit = pdd->lds_limit; + pAperture->gpuvm_base = pdd->gpuvm_base; + pAperture->gpuvm_limit = pdd->gpuvm_limit; + pAperture->scratch_base = pdd->scratch_base; + pAperture->scratch_limit = pdd->scratch_limit;

- /*if the process-device list isn't empty*/ - if (kfd_has_process_device_data(p)) { - /* Run over all pdd of the process */ - pdd = kfd_get_first_process_device_data(p); - do { - pAperture = - &args->process_apertures[args->num_of_nodes]; - pAperture->gpu_id = pdd->dev->id; - pAperture->lds_base = pdd->lds_base; - pAperture->lds_limit = pdd->lds_limit; - pAperture->gpuvm_base = pdd->gpuvm_base; - pAperture->gpuvm_limit = pdd->gpuvm_limit; - pAperture->scratch_base = pdd->scratch_base; - pAperture->scratch_limit = pdd->scratch_limit; - - dev_dbg(kfd_device, - "node id %u\n", args->num_of_nodes); - dev_dbg(kfd_device, - "gpu id %u\n", pdd->dev->id); - dev_dbg(kfd_device, - "lds_base %llX\n", pdd->lds_base); - dev_dbg(kfd_device, - "lds_limit %llX\n", pdd->lds_limit); - dev_dbg(kfd_device, - "gpuvm_base %llX\n", pdd->gpuvm_base); - dev_dbg(kfd_device, - "gpuvm_limit %llX\n", pdd->gpuvm_limit); - dev_dbg(kfd_device, - "scratch_base %llX\n", pdd->scratch_base); - dev_dbg(kfd_device, - "scratch_limit %llX\n", pdd->scratch_limit); - - args->num_of_nodes++; - - pdd = kfd_get_next_process_device_data(p, pdd); - } while (pdd && (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS)); - } + dev_dbg(kfd_device, + "node id %u\n", args->num_of_nodes); + dev_dbg(kfd_device, + "gpu id %u\n", pdd->dev->id); + dev_dbg(kfd_device, + "lds_base %llX\n", pdd->lds_base); + dev_dbg(kfd_device, + "lds_limit %llX\n", pdd->lds_limit); + dev_dbg(kfd_device, + "gpuvm_base %llX\n", pdd->gpuvm_base); + dev_dbg(kfd_device, + "gpuvm_limit %llX\n", pdd->gpuvm_limit); + dev_dbg(kfd_device, + "scratch_base %llX\n", pdd->scratch_base); + dev_dbg(kfd_device, + "scratch_limit %llX\n", pdd->scratch_limit);

+ if (++args->num_of_nodes >= NUM_OF_SUPPORTED_GPUS) + break; + } mutex_unlock(&p->mutex);

return 0; @@ -930,9 +925,8 @@ static int kfd_ioctl_get_process_apertures_new(struct file *filp, { struct kfd_ioctl_get_process_apertures_new_args *args = data; struct kfd_process_device_apertures *pa; - struct kfd_process_device *pdd; - uint32_t nodes = 0; int ret; + int i;

dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);

@@ -941,17 +935,7 @@ static int kfd_ioctl_get_process_apertures_new(struct file *filp, * sufficient memory */ mutex_lock(&p->mutex); - - if (!kfd_has_process_device_data(p)) - goto out_unlock; - - /* Run over all pdd of the process */ - pdd = kfd_get_first_process_device_data(p); - do { - args->num_of_nodes++; - pdd = kfd_get_next_process_device_data(p, pdd); - } while (pdd); - + args->num_of_nodes = p->n_pdds; goto out_unlock; }

@@ -966,22 +950,23 @@ static int kfd_ioctl_get_process_apertures_new(struct file *filp,

mutex_lock(&p->mutex);

- if (!kfd_has_process_device_data(p)) { + if (!p->n_pdds) { args->num_of_nodes = 0; kfree(pa); goto out_unlock; }

/* Run over all pdd of the process */ - pdd = kfd_get_first_process_device_data(p); - do { - pa[nodes].gpu_id = pdd->dev->id; - pa[nodes].lds_base = pdd->lds_base; - pa[nodes].lds_limit = pdd->lds_limit; - pa[nodes].gpuvm_base = pdd->gpuvm_base; - pa[nodes].gpuvm_limit = pdd->gpuvm_limit; - pa[nodes].scratch_base = pdd->scratch_base; - pa[nodes].scratch_limit = pdd->scratch_limit; + for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) { + struct kfd_process_device *pdd = p->pdds[i]; + + pa[i].gpu_id = pdd->dev->id; + pa[i].lds_base = pdd->lds_base; + pa[i].lds_limit = pdd->lds_limit; + pa[i].gpuvm_base = pdd->gpuvm_base; + pa[i].gpuvm_limit = pdd->gpuvm_limit; + pa[i].scratch_base = pdd->scratch_base; + pa[i].scratch_limit = pdd->scratch_limit;

dev_dbg(kfd_device, "gpu id %u\n", pdd->dev->id); @@ -997,17 +982,14 @@ static int kfd_ioctl_get_process_apertures_new(struct file *filp, "scratch_base %llX\n", pdd->scratch_base); dev_dbg(kfd_device, "scratch_limit %llX\n", pdd->scratch_limit); - nodes++; - - pdd = kfd_get_next_process_device_data(p, pdd); - } while (pdd && (nodes < args->num_of_nodes)); + } mutex_unlock(&p->mutex);

- args->num_of_nodes = nodes; + args->num_of_nodes = i; ret = copy_to_user( (void __user *)args->kfd_process_device_apertures_ptr, pa, - (nodes * sizeof(struct kfd_process_device_apertures))); + (i * sizeof(struct kfd_process_device_apertures))); kfree(pa); return ret ? -EFAULT : 0;

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c index 5a64915abaf7..1a266b78f0d8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c @@ -131,11 +131,11 @@ int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd) */ void kfd_iommu_unbind_process(struct kfd_process *p) { - struct kfd_process_device *pdd; + int i;

- list_for_each_entry(pdd, &p->per_device_data, per_device_list) - if (pdd->bound == PDD_BOUND) - amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid); + for (i = 0; i < p->n_pdds; i++) + if (p->pdds[i]->bound == PDD_BOUND) + amd_iommu_unbind_pasid(p->pdds[i]->dev->pdev, p->pasid); }

/* Callback for process shutdown invoked by the IOMMU driver */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index e2ebd5a1d4de..d9f8d3d48aac 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -45,6 +45,7 @@ #include <linux/swap.h>

#include "amd_shared.h" +#include "amdgpu.h"

#define KFD_MAX_RING_ENTRY_SIZE 8

@@ -644,12 +645,6 @@ enum kfd_pdd_bound {

/* Data that is per-process-per device. */ struct kfd_process_device { - /* - * List of all per-device data for a process. - * Starts from kfd_process.per_device_data. - */ - struct list_head per_device_list; - /* The device that owns this data. */ struct kfd_dev *dev;

@@ -766,10 +761,11 @@ struct kfd_process { uint16_t pasid;

/* - * List of kfd_process_device structures, + * Array of kfd_process_device pointers, * one for each device the process is using. */ - struct list_head per_device_data; + struct kfd_process_device *pdds[MAX_GPU_INSTANCE]; + uint32_t n_pdds;

struct process_queue_manager pqm;

@@ -867,14 +863,6 @@ void *kfd_process_device_translate_handle(struct kfd_process_device *p, void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, int handle);

-/* Process device data iterator */ -struct kfd_process_device *kfd_get_first_process_device_data( - struct kfd_process *p); -struct kfd_process_device *kfd_get_next_process_device_data( - struct kfd_process *p, - struct kfd_process_device *pdd); -bool kfd_has_process_device_data(struct kfd_process *p); - /* PASIDs */ int kfd_pasid_init(void); void kfd_pasid_exit(void); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 2807e1c4d59b..031e752e3154 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -505,7 +505,7 @@ static int kfd_sysfs_create_file(struct kfd_process *p, struct attribute *attr, static int kfd_procfs_add_sysfs_stats(struct kfd_process *p) { int ret = 0; - struct kfd_process_device *pdd; + int i; char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];

if (!p) @@ -520,7 +520,8 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p) * - proc/<pid>/stats_<gpuid>/evicted_ms * - proc/<pid>/stats_<gpuid>/cu_occupancy */ - list_for_each_entry(pdd, &p->per_device_data, per_device_list) { + for (i = 0; i < p->n_pdds; i++) { + struct kfd_process_device *pdd = p->pdds[i]; struct kobject *kobj_stats;

snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN, @@ -571,7 +572,7 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p) static int kfd_procfs_add_sysfs_files(struct kfd_process *p) { int ret = 0; - struct kfd_process_device *pdd; + int i;

if (!p) return -EINVAL; @@ -584,7 +585,9 @@ static int kfd_procfs_add_sysfs_files(struct kfd_process *p) * - proc/<pid>/vram_<gpuid> * - proc/<pid>/sdma_<gpuid> */ - list_for_each_entry(pdd, &p->per_device_data, per_device_list) { + for (i = 0; i < p->n_pdds; i++) { + struct kfd_process_device *pdd = p->pdds[i]; + snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u", pdd->dev->id); ret = kfd_sysfs_create_file(p, &pdd->attr_vram, pdd->vram_filename); @@ -875,21 +878,23 @@ void kfd_unref_process(struct kfd_process *p) kref_put(&p->ref, kfd_process_ref_release); }

+ static void kfd_process_device_free_bos(struct kfd_process_device *pdd) { struct kfd_process *p = pdd->process; void *mem; int id; + int i;

/* * Remove all handles from idr and release appropriate * local memory object */ idr_for_each_entry(&pdd->alloc_idr, mem, id) { - struct kfd_process_device *peer_pdd;

- list_for_each_entry(peer_pdd, &p->per_device_data, - per_device_list) { + for (i = 0; i < p->n_pdds; i++) { + struct kfd_process_device *peer_pdd = p->pdds[i]; + if (!peer_pdd->vm) continue; amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( @@ -903,18 +908,19 @@ static void kfd_process_device_free_bos(struct kfd_process_device *pdd)

static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p) { - struct kfd_process_device *pdd; + int i;

- list_for_each_entry(pdd, &p->per_device_data, per_device_list) - kfd_process_device_free_bos(pdd); + for (i = 0; i < p->n_pdds; i++) + kfd_process_device_free_bos(p->pdds[i]); }

static void kfd_process_destroy_pdds(struct kfd_process *p) { - struct kfd_process_device *pdd, *temp; + int i; + + for (i = 0; i < p->n_pdds; i++) { + struct kfd_process_device *pdd = p->pdds[i];

- list_for_each_entry_safe(pdd, temp, &p->per_device_data, - per_device_list) { pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n", pdd->dev->id, p->pasid);

@@ -927,8 +933,6 @@ static void kfd_process_destroy_pdds(struct kfd_process *p) amdgpu_amdkfd_gpuvm_destroy_process_vm( pdd->dev->kgd, pdd->vm);

- list_del(&pdd->per_device_list); - if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base) free_pages((unsigned long)pdd->qpd.cwsr_kaddr, get_order(KFD_CWSR_TBA_TMA_SIZE)); @@ -949,7 +953,9 @@ static void kfd_process_destroy_pdds(struct kfd_process *p) }

kfree(pdd); + p->pdds[i] = NULL; } + p->n_pdds = 0; }

/* No process locking is needed in this function, because the process @@ -961,7 +967,7 @@ static void kfd_process_wq_release(struct work_struct *work) { struct kfd_process *p = container_of(work, struct kfd_process, release_work); - struct kfd_process_device *pdd; + int i;

/* Remove the procfs files */ if (p->kobj) { @@ -970,7 +976,9 @@ static void kfd_process_wq_release(struct work_struct *work) kobject_put(p->kobj_queues); p->kobj_queues = NULL;

- list_for_each_entry(pdd, &p->per_device_data, per_device_list) { + for (i = 0; i < p->n_pdds; i++) { + struct kfd_process_device *pdd = p->pdds[i]; + sysfs_remove_file(p->kobj, &pdd->attr_vram); sysfs_remove_file(p->kobj, &pdd->attr_sdma); sysfs_remove_file(p->kobj, &pdd->attr_evict); @@ -1020,7 +1028,7 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn, struct mm_struct *mm) { struct kfd_process *p; - struct kfd_process_device *pdd = NULL; + int i;

/* * The kfd_process structure can not be free because the @@ -1044,8 +1052,8 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn, * pdd is in debug mode, we should first force unregistration, * then we will be able to destroy the queues */ - list_for_each_entry(pdd, &p->per_device_data, per_device_list) { - struct kfd_dev *dev = pdd->dev; + for (i = 0; i < p->n_pdds; i++) { + struct kfd_dev *dev = p->pdds[i]->dev;

mutex_lock(kfd_get_dbgmgr_mutex()); if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) { @@ -1081,11 +1089,11 @@ static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = { static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep) { unsigned long offset; - struct kfd_process_device *pdd; + int i;

- list_for_each_entry(pdd, &p->per_device_data, per_device_list) { - struct kfd_dev *dev = pdd->dev; - struct qcm_process_device *qpd = &pdd->qpd; + for (i = 0; i < p->n_pdds; i++) { + struct kfd_dev *dev = p->pdds[i]->dev; + struct qcm_process_device *qpd = &p->pdds[i]->qpd;

if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base) continue; @@ -1162,7 +1170,7 @@ static struct kfd_process *create_process(const struct task_struct *thread) mutex_init(&process->mutex); process->mm = thread->mm; process->lead_thread = thread->group_leader; - INIT_LIST_HEAD(&process->per_device_data); + process->n_pdds = 0; INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker); INIT_DELAYED_WORK(&process->restore_work, restore_process_worker); process->last_restore_timestamp = get_jiffies_64(); @@ -1244,11 +1252,11 @@ static int init_doorbell_bitmap(struct qcm_process_device *qpd, struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, struct kfd_process *p) { - struct kfd_process_device *pdd = NULL; + int i;

- list_for_each_entry(pdd, &p->per_device_data, per_device_list) - if (pdd->dev == dev) - return pdd; + for (i = 0; i < p->n_pdds; i++) + if (p->pdds[i]->dev == dev) + return p->pdds[i];

return NULL; } @@ -1258,6 +1266,8 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, { struct kfd_process_device *pdd = NULL;

+ if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE)) + return NULL; pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); if (!pdd) return NULL; @@ -1286,7 +1296,7 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, pdd->vram_usage = 0; pdd->sdma_past_activity_counter = 0; atomic64_set(&pdd->evict_duration_counter, 0); - list_add(&pdd->per_device_list, &p->per_device_data); + p->pdds[p->n_pdds++] = pdd;

/* Init idr used for memory handle translation */ idr_init(&pdd->alloc_idr); @@ -1418,28 +1428,6 @@ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, return ERR_PTR(err); }

-struct kfd_process_device *kfd_get_first_process_device_data( - struct kfd_process *p) -{ - return list_first_entry(&p->per_device_data, - struct kfd_process_device, - per_device_list); -} - -struct kfd_process_device *kfd_get_next_process_device_data( - struct kfd_process *p, - struct kfd_process_device *pdd) -{ - if (list_is_last(&pdd->per_device_list, &p->per_device_data)) - return NULL; - return list_next_entry(pdd, per_device_list); -} - -bool kfd_has_process_device_data(struct kfd_process *p) -{ - return !(list_empty(&p->per_device_data)); -} - /* Create specific handle mapped to mem from process local memory idr * Assumes that the process lock is held. */ @@ -1515,11 +1503,13 @@ struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm) */ int kfd_process_evict_queues(struct kfd_process *p) { - struct kfd_process_device *pdd; int r = 0; + int i; unsigned int n_evicted = 0;

- list_for_each_entry(pdd, &p->per_device_data, per_device_list) { + for (i = 0; i < p->n_pdds; i++) { + struct kfd_process_device *pdd = p->pdds[i]; + r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, &pdd->qpd); if (r) { @@ -1535,7 +1525,9 @@ int kfd_process_evict_queues(struct kfd_process *p) /* To keep state consistent, roll back partial eviction by * restoring queues */ - list_for_each_entry(pdd, &p->per_device_data, per_device_list) { + for (i = 0; i < p->n_pdds; i++) { + struct kfd_process_device *pdd = p->pdds[i]; + if (n_evicted == 0) break; if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, @@ -1551,10 +1543,12 @@ int kfd_process_evict_queues(struct kfd_process *p) /* kfd_process_restore_queues - Restore all user queues of a process */ int kfd_process_restore_queues(struct kfd_process *p) { - struct kfd_process_device *pdd; int r, ret = 0; + int i; + + for (i = 0; i < p->n_pdds; i++) { + struct kfd_process_device *pdd = p->pdds[i];

- list_for_each_entry(pdd, &p->per_device_data, per_device_list) { r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, &pdd->qpd); if (r) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index eb1635ac8988..95a6c36cea4c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -126,10 +126,10 @@ int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,

void kfd_process_dequeue_from_all_devices(struct kfd_process *p) { - struct kfd_process_device *pdd; + int i;

- list_for_each_entry(pdd, &p->per_device_data, per_device_list) - kfd_process_dequeue_from_device(pdd); + for (i = 0; i < p->n_pdds; i++) + kfd_process_dequeue_from_device(p->pdds[i]); }

int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)

From: Philip Yang Philip.Yang@amd.com

Add svm (shared virtual memory) ioctl data structure and API definition.

The svm ioctl API is designed to be extensible in the future. All operations are provided by a single IOCTL to preserve ioctl number space. The arguments structure ends with a variable size array of attributes that can be used to set or get one or multiple attributes.

Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 7 ++ include/uapi/linux/kfd_ioctl.h | 128 ++++++++++++++++++++++- 2 files changed, 133 insertions(+), 2 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 8c87afce12df..c5288a6e45b9 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1746,6 +1746,11 @@ static int kfd_ioctl_smi_events(struct file *filep, return kfd_smi_event_open(dev, &args->anon_fd); }

+static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data) +{ + return -EINVAL; +} + #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \ .cmd_drv = 0, .name = #ioctl} @@ -1844,6 +1849,8 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {

AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS, kfd_ioctl_smi_events, 0), + + AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0), };

#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index 695b606da4b1..5d4a4b3e0b61 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -29,9 +29,10 @@ /* * - 1.1 - initial version * - 1.3 - Add SMI events support + * - 1.4 - Add SVM API */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 3 +#define KFD_IOCTL_MINOR_VERSION 4

struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */ @@ -471,6 +472,127 @@ enum kfd_mmio_remap { KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4, };

+/* Guarantee host access to memory */ +#define KFD_IOCTL_SVM_FLAG_HOST_ACCESS 0x00000001 +/* Fine grained coherency between all devices with access */ +#define KFD_IOCTL_SVM_FLAG_COHERENT 0x00000002 +/* Use any GPU in same hive as preferred device */ +#define KFD_IOCTL_SVM_FLAG_HIVE_LOCAL 0x00000004 +/* GPUs only read, allows replication */ +#define KFD_IOCTL_SVM_FLAG_GPU_RO 0x00000008 +/* Allow execution on GPU */ +#define KFD_IOCTL_SVM_FLAG_GPU_EXEC 0x00000010 + +/** + * kfd_ioctl_svm_op - SVM ioctl operations + * + * @KFD_IOCTL_SVM_OP_SET_ATTR: Modify one or more attributes + * @KFD_IOCTL_SVM_OP_GET_ATTR: Query one or more attributes + */ +enum kfd_ioctl_svm_op { + KFD_IOCTL_SVM_OP_SET_ATTR, + KFD_IOCTL_SVM_OP_GET_ATTR +}; + +/** kfd_ioctl_svm_location - Enum for preferred and prefetch locations + * + * GPU IDs are used to specify GPUs as preferred and prefetch locations. + * Below definitions are used for system memory or for leaving the preferred + * location unspecified. + */ +enum kfd_ioctl_svm_location { + KFD_IOCTL_SVM_LOCATION_SYSMEM = 0, + KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff +}; + +/** + * kfd_ioctl_svm_attr_type - SVM attribute types + * + * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: gpuid of the preferred location, 0 for + * system memory + * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: gpuid of the prefetch location, 0 for + * system memory. Setting this triggers an + * immediate prefetch (migration). + * @KFD_IOCTL_SVM_ATTR_ACCESS: + * @KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: + * @KFD_IOCTL_SVM_ATTR_NO_ACCESS: specify memory access for the gpuid given + * by the attribute value + * @KFD_IOCTL_SVM_ATTR_SET_FLAGS: bitmask of flags to set (see + * KFD_IOCTL_SVM_FLAG_...) + * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS: bitmask of flags to clear + * @KFD_IOCTL_SVM_ATTR_GRANULARITY: migration granularity + * (log2 num pages) + */ +enum kfd_ioctl_svm_attr_type { + KFD_IOCTL_SVM_ATTR_PREFERRED_LOC, + KFD_IOCTL_SVM_ATTR_PREFETCH_LOC, + KFD_IOCTL_SVM_ATTR_ACCESS, + KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE, + KFD_IOCTL_SVM_ATTR_NO_ACCESS, + KFD_IOCTL_SVM_ATTR_SET_FLAGS, + KFD_IOCTL_SVM_ATTR_CLR_FLAGS, + KFD_IOCTL_SVM_ATTR_GRANULARITY +}; + +/** + * kfd_ioctl_svm_attribute - Attributes as pairs of type and value + * + * The meaning of the @value depends on the attribute type. + * + * @type: attribute type (see enum @kfd_ioctl_svm_attr_type) + * @value: attribute value + */ +struct kfd_ioctl_svm_attribute { + __u32 type; + __u32 value; +}; + +/** + * kfd_ioctl_svm_args - Arguments for SVM ioctl + * + * @op specifies the operation to perform (see enum + * @kfd_ioctl_svm_op). @start_addr and @size are common for all + * operations. + * + * A variable number of attributes can be given in @attrs. + * @nattr specifies the number of attributes. New attributes can be + * added in the future without breaking the ABI. If unknown attributes + * are given, the function returns -EINVAL. + * + * @KFD_IOCTL_SVM_OP_SET_ATTR sets attributes for a virtual address + * range. It may overlap existing virtual address ranges. If it does, + * the existing ranges will be split such that the attribute changes + * only apply to the specified address range. + * + * @KFD_IOCTL_SVM_OP_GET_ATTR returns the intersection of attributes + * over all memory in the given range and returns the result as the + * attribute value. If different pages have different preferred or + * prefetch locations, 0xffffffff will be returned for + * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC or + * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC resepctively. For + * @KFD_IOCTL_SVM_ATTR_SET_FLAGS, flags of all pages will be + * aggregated by bitwise AND. The minimum migration granularity + * throughout the range will be returned for + * @KFD_IOCTL_SVM_ATTR_GRANULARITY. + * + * Querying of accessibility attributes works by initializing the + * attribute type to @KFD_IOCTL_SVM_ATTR_ACCESS and the value to the + * GPUID being queried. Multiple attributes can be given to allow + * querying multiple GPUIDs. The ioctl function overwrites the + * attribute type to indicate the access for the specified GPU. + * + * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS is invalid for + * @KFD_IOCTL_SVM_OP_GET_ATTR. + */ +struct kfd_ioctl_svm_args { + __u64 start_addr; + __u64 size; + __u32 op; + __u32 nattr; + /* Variable length array of attributes */ + struct kfd_ioctl_svm_attribute attrs[0]; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -571,7 +693,9 @@ enum kfd_mmio_remap { #define AMDKFD_IOC_SMI_EVENTS \ AMDKFD_IOWR(0x1F, struct kfd_ioctl_smi_events_args)

+#define AMDKFD_IOC_SVM AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x20 +#define AMDKFD_COMMAND_END 0x21

#endif

From: Philip Yang Philip.Yang@amd.com

svm range structure stores the range start address, size, attributes, flags, prefetch location and gpu bitmap which indicates which GPU this range maps to. Same virtual address is shared by CPU and GPUs.

Process has svm range list which uses both interval tree and list to store all svm ranges registered by the process. Interval tree is used by GPU vm fault handler and CPU page fault handler to get svm range structure from the specific address. List is used to scan all ranges in eviction restore work.

Apply attributes preferred location, prefetch location, mapping flags, migration granularity to svm range, store mapping gpu index into bitmap.

Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/Makefile | 3 +- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 21 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 14 + drivers/gpu/drm/amd/amdkfd/kfd_process.c | 9 + drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 603 +++++++++++++++++++++++ drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 93 ++++ 6 files changed, 741 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_svm.c create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_svm.h

diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile index e1e4115dcf78..387ce0217d35 100644 --- a/drivers/gpu/drm/amd/amdkfd/Makefile +++ b/drivers/gpu/drm/amd/amdkfd/Makefile @@ -54,7 +54,8 @@ AMDKFD_FILES := $(AMDKFD_PATH)/kfd_module.o \ $(AMDKFD_PATH)/kfd_dbgdev.o \ $(AMDKFD_PATH)/kfd_dbgmgr.o \ $(AMDKFD_PATH)/kfd_smi_events.o \ - $(AMDKFD_PATH)/kfd_crat.o + $(AMDKFD_PATH)/kfd_crat.o \ + $(AMDKFD_PATH)/kfd_svm.o

ifneq ($(CONFIG_AMD_IOMMU_V2),) AMDKFD_FILES += $(AMDKFD_PATH)/kfd_iommu.o diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index c5288a6e45b9..2d3ba7e806d5 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -38,6 +38,7 @@ #include "kfd_priv.h" #include "kfd_device_queue_manager.h" #include "kfd_dbgmgr.h" +#include "kfd_svm.h" #include "amdgpu_amdkfd.h" #include "kfd_smi_events.h"

@@ -1748,7 +1749,25 @@ static int kfd_ioctl_smi_events(struct file *filep,

static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data) { - return -EINVAL; + struct kfd_ioctl_svm_args *args = data; + int r = 0; + + pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n", + args->start_addr, args->size, args->op, args->nattr); + + if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK)) + return -EINVAL; + if (!args->start_addr || !args->size) + return -EINVAL; + + mutex_lock(&p->mutex); + + r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr, + args->attrs); + + mutex_unlock(&p->mutex); + + return r; }

#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 4ef8804adcf5..cbb2bae1982d 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -726,6 +726,17 @@ struct kfd_process_device {

#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)

+struct svm_range_list { + struct mutex lock; /* use svms_lock/unlock(svms) */; + unsigned int saved_flags; + struct rb_root_cached objects; + struct list_head list; + struct srcu_struct srcu; + struct work_struct srcu_free_work; + struct list_head free_list; + struct mutex free_list_lock; +}; + /* Process data */ struct kfd_process { /* @@ -804,6 +815,9 @@ struct kfd_process { struct kobject *kobj; struct kobject *kobj_queues; struct attribute attr_pasid; + + /* shared virtual memory registered by this process */ + struct svm_range_list svms; };

#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 7396f3a6d0ee..791f17308b1b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -35,6 +35,7 @@ #include <linux/pm_runtime.h> #include "amdgpu_amdkfd.h" #include "amdgpu.h" +#include "kfd_svm.h"

struct mm_struct;

@@ -42,6 +43,7 @@ struct mm_struct; #include "kfd_device_queue_manager.h" #include "kfd_dbgmgr.h" #include "kfd_iommu.h" +#include "kfd_svm.h"

/* * List of struct kfd_process (field kfd_process). @@ -997,6 +999,7 @@ static void kfd_process_wq_release(struct work_struct *work) kfd_iommu_unbind_process(p);

kfd_process_free_outstanding_kfd_bos(p); + svm_range_list_fini(p);

kfd_process_destroy_pdds(p); dma_fence_put(p->ef); @@ -1190,6 +1193,10 @@ static struct kfd_process *create_process(const struct task_struct *thread) if (err != 0) goto err_init_apertures;

+ err = svm_range_list_init(process); + if (err) + goto err_init_svm_range_list; + /* Must be last, have to use release destruction after this */ process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops; err = mmu_notifier_register(&process->mmu_notifier, process->mm); @@ -1203,6 +1210,8 @@ static struct kfd_process *create_process(const struct task_struct *thread) return process;

err_register_notifier: + svm_range_list_fini(process); +err_init_svm_range_list: kfd_process_free_outstanding_kfd_bos(process); kfd_process_destroy_pdds(process); err_init_apertures: diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c new file mode 100644 index 000000000000..0b0410837be9 --- /dev/null +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -0,0 +1,603 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <linux/types.h> +#include "amdgpu_sync.h" +#include "amdgpu_object.h" +#include "amdgpu_vm.h" +#include "amdgpu_mn.h" +#include "kfd_priv.h" +#include "kfd_svm.h" + +/** + * svm_range_unlink - unlink svm_range from lists and interval tree + * @prange: svm range structure to be removed + * + * Remove the svm range from svms interval tree and link list + * + * Context: The caller must hold svms_lock + */ +static void svm_range_unlink(struct svm_range *prange) +{ + pr_debug("prange 0x%p [0x%lx 0x%lx]\n", prange, prange->it_node.start, + prange->it_node.last); + + list_del_rcu(&prange->list); + interval_tree_remove(&prange->it_node, &prange->svms->objects); +} + +/** + * svm_range_add_to_svms - add svm range to svms + * @prange: svm range structure to be added + * + * Add the svm range to svms interval tree and link list + * + * Context: The caller must hold svms_lock + */ +static void svm_range_add_to_svms(struct svm_range *prange) +{ + pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + + list_add_tail_rcu(&prange->list, &prange->svms->list); + interval_tree_insert(&prange->it_node, &prange->svms->objects); +} + +static void svm_range_remove(struct svm_range *prange) +{ + pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + + kvfree(prange->pages_addr); + kfree(prange); +} + +static void +svm_range_set_default_attributes(int32_t *location, int32_t *prefetch_loc, + uint8_t *granularity, uint32_t *flags) +{ + *location = 0; + *prefetch_loc = 0; + *granularity = 9; + *flags = + KFD_IOCTL_SVM_FLAG_HOST_ACCESS | KFD_IOCTL_SVM_FLAG_COHERENT; +} + +static struct +svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start, + uint64_t last) +{ + uint64_t size = last - start + 1; + struct svm_range *prange; + + prange = kzalloc(sizeof(*prange), GFP_KERNEL); + if (!prange) + return NULL; + prange->npages = size; + prange->svms = svms; + prange->it_node.start = start; + prange->it_node.last = last; + INIT_LIST_HEAD(&prange->list); + INIT_LIST_HEAD(&prange->update_list); + INIT_LIST_HEAD(&prange->remove_list); + svm_range_set_default_attributes(&prange->preferred_loc, + &prange->prefetch_loc, + &prange->granularity, &prange->flags); + + pr_debug("svms 0x%p [0x%llx 0x%llx]\n", svms, start, last); + + return prange; +} + +static struct kfd_dev * +svm_get_supported_dev_by_id(struct kfd_process *p, uint32_t gpu_id, + int *r_gpuidx) +{ + struct kfd_dev *dev; + int gpuidx; + int r; + + gpuidx = kfd_process_gpuidx_from_gpuid(p, gpu_id); + if (gpuidx < 0) { + pr_debug("failed to get device by id 0x%x\n", gpu_id); + return NULL; + } + r = kfd_process_device_from_gpuidx(p, gpuidx, &dev); + if (r < 0) { + pr_debug("failed to get device by idx 0x%x\n", gpuidx); + return NULL; + } + if (dev->device_info->asic_family < CHIP_VEGA10) { + pr_debug("device id 0x%x does not support SVM\n", gpu_id); + return NULL; + } + if (r_gpuidx) + *r_gpuidx = gpuidx; + return dev; +} + +static int +svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange, + uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs) +{ + uint32_t i; + int gpuidx; + + for (i = 0; i < nattr; i++) { + switch (attrs[i].type) { + case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: + if (attrs[i].value != KFD_IOCTL_SVM_LOCATION_SYSMEM && + attrs[i].value != KFD_IOCTL_SVM_LOCATION_UNDEFINED && + !svm_get_supported_dev_by_id(p, attrs[i].value, NULL)) + return -EINVAL; + prange->preferred_loc = attrs[i].value; + break; + case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: + if (attrs[i].value != KFD_IOCTL_SVM_LOCATION_SYSMEM && + !svm_get_supported_dev_by_id(p, attrs[i].value, NULL)) + return -EINVAL; + prange->prefetch_loc = attrs[i].value; + break; + case KFD_IOCTL_SVM_ATTR_ACCESS: + case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: + case KFD_IOCTL_SVM_ATTR_NO_ACCESS: + if (!svm_get_supported_dev_by_id(p, attrs[i].value, + &gpuidx)) + return -EINVAL; + if (attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS) { + bitmap_clear(prange->bitmap_access, gpuidx, 1); + bitmap_clear(prange->bitmap_aip, gpuidx, 1); + } else if (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS) { + bitmap_set(prange->bitmap_access, gpuidx, 1); + bitmap_clear(prange->bitmap_aip, gpuidx, 1); + } else { + bitmap_clear(prange->bitmap_access, gpuidx, 1); + bitmap_set(prange->bitmap_aip, gpuidx, 1); + } + break; + case KFD_IOCTL_SVM_ATTR_SET_FLAGS: + prange->flags |= attrs[i].value; + break; + case KFD_IOCTL_SVM_ATTR_CLR_FLAGS: + prange->flags &= ~attrs[i].value; + break; + case KFD_IOCTL_SVM_ATTR_GRANULARITY: + prange->granularity = attrs[i].value; + break; + default: + pr_debug("unknown attr type 0x%x\n", attrs[i].type); + return -EINVAL; + } + } + + return 0; +} + +/** + * svm_range_debug_dump - print all range information from svms + * @svms: svm range list header + * + * debug output svm range start, end, pages_addr, prefetch location from svms + * interval tree and link list + * + * Context: The caller must hold svms_lock + */ +static void svm_range_debug_dump(struct svm_range_list *svms) +{ + struct interval_tree_node *node; + struct svm_range *prange; + + pr_debug("dump svms 0x%p list\n", svms); + pr_debug("range\tstart\tpage\tend\t\tpages_addr\tlocation\n"); + + /* Not using list_for_each_entry_rcu because the caller is holding the + * svms lock + */ + list_for_each_entry(prange, &svms->list, list) { + pr_debug("0x%lx\t0x%llx\t0x%llx\t0x%llx\t0x%x\n", + prange->it_node.start, prange->npages, + prange->it_node.start + prange->npages - 1, + prange->pages_addr ? *prange->pages_addr : 0, + prange->actual_loc); + } + + pr_debug("dump svms 0x%p interval tree\n", svms); + pr_debug("range\tstart\tpage\tend\t\tpages_addr\tlocation\n"); + node = interval_tree_iter_first(&svms->objects, 0, ~0ULL); + while (node) { + prange = container_of(node, struct svm_range, it_node); + pr_debug("0x%lx\t0x%llx\t0x%llx\t0x%llx\t0x%x\n", + prange->it_node.start, prange->npages, + prange->it_node.start + prange->npages - 1, + prange->pages_addr ? *prange->pages_addr : 0, + prange->actual_loc); + node = interval_tree_iter_next(node, 0, ~0ULL); + } +} + +/** + * svm_range_handle_overlap - split overlap ranges + * @svms: svm range list header + * @new: range added with this attributes + * @start: range added start address, in pages + * @last: range last address, in pages + * @update_list: output, the ranges attributes are updated. For set_attr, this + * will do validation and map to GPUs. For unmap, this will be + * removed and unmap from GPUs + * @insert_list: output, the ranges will be inserted into svms, attributes are + * not changes. For set_attr, this will add into svms. For unmap, + * will remove duplicate range from update_list because it is + * unmapped, should not insert to svms. + * @remove_list:output, the ranges will be removed from svms + * @left: the remaining range after overlap, For set_attr, this will be added + * as new range. For unmap, this is ignored. + * + * Total have 5 overlap cases. + * + * Context: The caller must hold svms_lock + */ +static int +svm_range_handle_overlap(struct svm_range_list *svms, struct svm_range *new, + unsigned long start, unsigned long last, + struct list_head *update_list, + struct list_head *insert_list, + struct list_head *remove_list, + unsigned long *left) +{ + struct interval_tree_node *node; + struct svm_range *prange; + struct svm_range *tmp; + int r = 0; + + INIT_LIST_HEAD(update_list); + INIT_LIST_HEAD(insert_list); + INIT_LIST_HEAD(remove_list); + + node = interval_tree_iter_first(&svms->objects, start, last); + while (node) { + struct interval_tree_node *next; + + pr_debug("found overlap node [0x%lx 0x%lx]\n", node->start, + node->last); + + prange = container_of(node, struct svm_range, it_node); + next = interval_tree_iter_next(node, start, last); + + if (node->start < start && node->last > last) { + pr_debug("split in 2 ranges\n"); + start = last + 1; + + } else if (node->start < start) { + /* + * For node->last == last, will exit loop + * for node->last < last, will continue in next loop + */ + uint64_t old_last = node->last; + + start = old_last + 1; + + } else if (node->start == start && node->last > last) { + pr_debug("change old range start\n"); + + start = last + 1; + + } else if (node->start == start) { + if (prange->it_node.last == last) + pr_debug("found exactly same range\n"); + else + pr_debug("next loop to add remaining range\n"); + + start = node->last + 1; + + } else { /* node->start > start */ + pr_debug("add new range at front\n"); + + start = node->last + 1; + } + + if (r) + goto out; + + node = next; + } + + if (left && start <= last) + *left = last - start + 1; + +out: + if (r) + list_for_each_entry_safe(prange, tmp, insert_list, list) + svm_range_remove(prange); + + return r; +} + +static void svm_range_srcu_free_work(struct work_struct *work_struct) +{ + struct svm_range_list *svms; + struct svm_range *prange; + struct svm_range *tmp; + + svms = container_of(work_struct, struct svm_range_list, srcu_free_work); + + synchronize_srcu(&svms->srcu); + + mutex_lock(&svms->free_list_lock); + list_for_each_entry_safe(prange, tmp, &svms->free_list, remove_list) { + list_del(&prange->remove_list); + svm_range_remove(prange); + } + mutex_unlock(&svms->free_list_lock); +} + +void svm_range_list_fini(struct kfd_process *p) +{ + pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms); + + /* Ensure srcu free work is finished before process is destroyed */ + flush_work(&p->svms.srcu_free_work); + cleanup_srcu_struct(&p->svms.srcu); + mutex_destroy(&p->svms.free_list_lock); +} + +int svm_range_list_init(struct kfd_process *p) +{ + struct svm_range_list *svms = &p->svms; + int r; + + svms->objects = RB_ROOT_CACHED; + mutex_init(&svms->lock); + INIT_LIST_HEAD(&svms->list); + r = init_srcu_struct(&svms->srcu); + if (r) { + pr_debug("failed %d to init srcu\n", r); + return r; + } + INIT_WORK(&svms->srcu_free_work, svm_range_srcu_free_work); + INIT_LIST_HEAD(&svms->free_list); + mutex_init(&svms->free_list_lock); + + return 0; +} + +/** + * svm_range_is_valid - check if virtual address range is valid + * @mm: current process mm_struct + * @start: range start address, in pages + * @size: range size, in pages + * + * Valid virtual address range means it belongs to one or more VMAs + * + * Context: Process context + * + * Return: + * true - valid svm range + * false - invalid svm range + */ +static bool +svm_range_is_valid(struct mm_struct *mm, uint64_t start, uint64_t size) +{ + const unsigned long device_vma = VM_IO | VM_PFNMAP | VM_MIXEDMAP; + struct vm_area_struct *vma; + unsigned long end; + + start <<= PAGE_SHIFT; + end = start + (size << PAGE_SHIFT); + + do { + vma = find_vma(mm, start); + if (!vma || start < vma->vm_start || + (vma->vm_flags & device_vma)) + return false; + start = min(end, vma->vm_end); + } while (start < end); + + return true; +} + +/** + * svm_range_add - add svm range and handle overlap + * @p: the range add to this process svms + * @start: page size aligned + * @size: page size aligned + * @nattr: number of attributes + * @attrs: array of attributes + * @update_list: output, the ranges need validate and update GPU mapping + * @insert_list: output, the ranges need insert to svms + * @remove_list: output, the ranges are replaced and need remove from svms + * + * Check if the virtual address range has overlap with the registered ranges, + * split the overlapped range, copy and adjust pages address and vram nodes in + * old and new ranges. + * + * Context: Process context, takes and releases svms_lock + * + * Return: + * 0 - OK, otherwise error code + */ +static int +svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size, + uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs, + struct list_head *update_list, struct list_head *insert_list, + struct list_head *remove_list) +{ + uint64_t last = start + size - 1UL; + struct svm_range_list *svms; + struct svm_range new = {0}; + struct svm_range *prange; + unsigned long left = 0; + int r = 0; + + pr_debug("svms 0x%p [0x%llx 0x%llx]\n", &p->svms, start, last); + + r = svm_range_apply_attrs(p, &new, nattr, attrs); + if (r) + return r; + + svms = &p->svms; + + r = svm_range_handle_overlap(svms, &new, start, last, update_list, + insert_list, remove_list, &left); + if (r) + return r; + + if (left) { + prange = svm_range_new(svms, last - left + 1, last); + list_add(&prange->list, insert_list); + list_add(&prange->update_list, update_list); + } + + return 0; +} + +static int +svm_range_set_attr(struct kfd_process *p, uint64_t start, uint64_t size, + uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs) +{ + struct amdkfd_process_info *process_info = p->kgd_process_info; + struct mm_struct *mm = current->mm; + struct list_head update_list; + struct list_head insert_list; + struct list_head remove_list; + struct svm_range_list *svms; + struct svm_range *prange; + struct svm_range *tmp; + int srcu_idx; + int r = 0; + + pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] pages 0x%llx\n", + p->pasid, &p->svms, start, start + size - 1, size); + + mmap_read_lock(mm); + if (!svm_range_is_valid(mm, start, size)) { + pr_debug("invalid range\n"); + mmap_read_unlock(mm); + return -EFAULT; + } + mmap_read_unlock(mm); + + mutex_lock(&process_info->lock); + + svms = &p->svms; + svms_lock(svms); + + r = svm_range_add(p, start, size, nattr, attrs, &update_list, + &insert_list, &remove_list); + if (r) { + svms_unlock(svms); + mutex_unlock(&process_info->lock); + return r; + } + + list_for_each_entry_safe(prange, tmp, &insert_list, list) + svm_range_add_to_svms(prange); + + /* Hold read lock to prevent prange is removed after unlocking svms */ + srcu_idx = srcu_read_lock(&svms->srcu); + svms_unlock(svms); + + /* Hold mm->map_sem and check if svm range is unmapped in parallel */ + mmap_read_lock(mm); + + if (!svm_range_is_valid(mm, start, size)) { + pr_debug("range is unmapped\n"); + mmap_read_unlock(mm); + srcu_read_unlock(&svms->srcu, srcu_idx); + r = -EFAULT; + goto out_remove; + } + + list_for_each_entry(prange, &update_list, update_list) { + + r = svm_range_apply_attrs(p, prange, nattr, attrs); + if (r) { + pr_debug("failed %d to apply attrs\n", r); + mmap_read_unlock(mm); + srcu_read_unlock(&prange->svms->srcu, srcu_idx); + goto out_remove; + } + } + + srcu_read_unlock(&svms->srcu, srcu_idx); + svms_lock(svms); + + mutex_lock(&svms->free_list_lock); + list_for_each_entry_safe(prange, tmp, &remove_list, remove_list) { + pr_debug("remove overlap prange svms 0x%p [0x%lx 0x%lx]\n", + prange->svms, prange->it_node.start, + prange->it_node.last); + svm_range_unlink(prange); + + pr_debug("schedule to free prange svms 0x%p [0x%lx 0x%lx]\n", + prange->svms, prange->it_node.start, + prange->it_node.last); + list_add_tail(&prange->remove_list, &svms->free_list); + } + if (!list_empty(&svms->free_list)) + schedule_work(&svms->srcu_free_work); + mutex_unlock(&svms->free_list_lock); + + svm_range_debug_dump(svms); + + svms_unlock(svms); + mmap_read_unlock(mm); + mutex_unlock(&process_info->lock); + + pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] done\n", p->pasid, + &p->svms, start, start + size - 1); + + return 0; + +out_remove: + svms_lock(svms); + list_for_each_entry_safe(prange, tmp, &insert_list, list) { + svm_range_unlink(prange); + list_add_tail(&prange->remove_list, &svms->free_list); + } + if (!list_empty(&svms->free_list)) + schedule_work(&svms->srcu_free_work); + svms_unlock(svms); + mutex_unlock(&process_info->lock); + + return r; +} + +int +svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start, + uint64_t size, uint32_t nattrs, struct kfd_ioctl_svm_attribute *attrs) +{ + int r; + + start >>= PAGE_SHIFT; + size >>= PAGE_SHIFT; + + switch (op) { + case KFD_IOCTL_SVM_OP_SET_ATTR: + r = svm_range_set_attr(p, start, size, nattrs, attrs); + break; + default: + r = EINVAL; + break; + } + + return r; +} diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h new file mode 100644 index 000000000000..c7c54fb73dfb --- /dev/null +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef KFD_SVM_H_ +#define KFD_SVM_H_ + +#include <linux/rwsem.h> +#include <linux/list.h> +#include <linux/mutex.h> +#include <linux/sched/mm.h> +#include <linux/hmm.h> +#include "amdgpu.h" +#include "kfd_priv.h" + +/** + * struct svm_range - shared virtual memory range + * + * @svms: list of svm ranges, structure defined in kfd_process + * @it_node: node [start, last] stored in interval tree, start, last are page + * aligned, page size is (last - start + 1) + * @list: link list node, used to scan all ranges of svms + * @update_list:link list node used to add to update_list + * @remove_list:link list node used to add to remove list + * @npages: number of pages + * @pages_addr: list of system memory physical page address + * @flags: flags defined as KFD_IOCTL_SVM_FLAG_* + * @perferred_loc: perferred location, 0 for CPU, or GPU id + * @perfetch_loc: last prefetch location, 0 for CPU, or GPU id + * @actual_loc: the actual location, 0 for CPU, or GPU id + * @granularity:migration granularity, log2 num pages + * @bitmap_access: index bitmap of GPUs which can access the range + * @bitmap_aip: index bitmap of GPUs which can access the range in place + * + * Data structure for virtual memory range shared by CPU and GPUs, it can be + * allocated from system memory ram or device vram, and migrate from ram to vram + * or from vram to ram. + */ +struct svm_range { + struct svm_range_list *svms; + struct interval_tree_node it_node; + struct list_head list; + struct list_head update_list; + struct list_head remove_list; + uint64_t npages; + dma_addr_t *pages_addr; + uint32_t flags; + uint32_t preferred_loc; + uint32_t prefetch_loc; + uint32_t actual_loc; + uint8_t granularity; + DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE); + DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE); +}; + +static inline void svms_lock(struct svm_range_list *svms) +{ + mutex_lock(&svms->lock); + svms->saved_flags = memalloc_nofs_save(); + +} +static inline void svms_unlock(struct svm_range_list *svms) +{ + memalloc_nofs_restore(svms->saved_flags); + mutex_unlock(&svms->lock); +} + +int svm_range_list_init(struct kfd_process *p); +void svm_range_list_fini(struct kfd_process *p); +int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start, + uint64_t size, uint32_t nattrs, + struct kfd_ioctl_svm_attribute *attrs); + +#endif /* KFD_SVM_H_ */

From: Philip Yang Philip.Yang@amd.com

Move the HMM get pages function from amdgpu_ttm and to amdgpu_mn. This common function will be used by new svm APIs.

Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c | 83 +++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h | 7 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 76 +++------------------- 3 files changed, 100 insertions(+), 66 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c index 828b5167ff12..997da4237a10 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c @@ -155,3 +155,86 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo) mmu_interval_notifier_remove(&bo->notifier); bo->notifier.mm = NULL; } + +int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, + struct mm_struct *mm, struct page **pages, + uint64_t start, uint64_t npages, + struct hmm_range **phmm_range, bool readonly, + bool mmap_locked) +{ + struct hmm_range *hmm_range; + unsigned long timeout; + unsigned long i; + unsigned long *pfns; + int r = 0; + + hmm_range = kzalloc(sizeof(*hmm_range), GFP_KERNEL); + if (unlikely(!hmm_range)) + return -ENOMEM; + + pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); + if (unlikely(!pfns)) { + r = -ENOMEM; + goto out_free_range; + } + + hmm_range->notifier = notifier; + hmm_range->default_flags = HMM_PFN_REQ_FAULT; + if (!readonly) + hmm_range->default_flags |= HMM_PFN_REQ_WRITE; + hmm_range->hmm_pfns = pfns; + hmm_range->start = start; + hmm_range->end = start + npages * PAGE_SIZE; + timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); + +retry: + hmm_range->notifier_seq = mmu_interval_read_begin(notifier); + + if (likely(!mmap_locked)) + mmap_read_lock(mm); + + r = hmm_range_fault(hmm_range); + + if (likely(!mmap_locked)) + mmap_read_unlock(mm); + if (unlikely(r)) { + /* + * FIXME: This timeout should encompass the retry from + * mmu_interval_read_retry() as well. + */ + if (r == -EBUSY && !time_after(jiffies, timeout)) + goto retry; + goto out_free_pfns; + } + + /* + * Due to default_flags, all pages are HMM_PFN_VALID or + * hmm_range_fault() fails. FIXME: The pages cannot be touched outside + * the notifier_lock, and mmu_interval_read_retry() must be done first. + */ + for (i = 0; pages && i < npages; i++) + pages[i] = hmm_pfn_to_page(pfns[i]); + + *phmm_range = hmm_range; + + return 0; + +out_free_pfns: + kvfree(pfns); +out_free_range: + kfree(hmm_range); + + return r; +} + +int amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range) +{ + int r; + + r = mmu_interval_read_retry(hmm_range->notifier, + hmm_range->notifier_seq); + kvfree(hmm_range->hmm_pfns); + kfree(hmm_range); + + return r; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h index a292238f75eb..7f7d37a457c3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h @@ -30,6 +30,13 @@ #include <linux/workqueue.h> #include <linux/interval_tree.h>

+int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier, + struct mm_struct *mm, struct page **pages, + uint64_t start, uint64_t npages, + struct hmm_range **phmm_range, bool readonly, + bool mmap_locked); +int amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range); + #if defined(CONFIG_HMM_MIRROR) int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr); void amdgpu_mn_unregister(struct amdgpu_bo *bo); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index aaad9e304ad9..f423f42cb9b5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -32,7 +32,6 @@

#include <linux/dma-mapping.h> #include <linux/iommu.h> -#include <linux/hmm.h> #include <linux/pagemap.h> #include <linux/sched/task.h> #include <linux/sched/mm.h> @@ -843,10 +842,8 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages) struct amdgpu_ttm_tt *gtt = (void *)ttm; unsigned long start = gtt->userptr; struct vm_area_struct *vma; - struct hmm_range *range; - unsigned long timeout; struct mm_struct *mm; - unsigned long i; + bool readonly; int r = 0;

mm = bo->notifier.mm; @@ -862,76 +859,26 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages) if (!mmget_not_zero(mm)) /* Happens during process shutdown */ return -ESRCH;

- range = kzalloc(sizeof(*range), GFP_KERNEL); - if (unlikely(!range)) { - r = -ENOMEM; - goto out; - } - range->notifier = &bo->notifier; - range->start = bo->notifier.interval_tree.start; - range->end = bo->notifier.interval_tree.last + 1; - range->default_flags = HMM_PFN_REQ_FAULT; - if (!amdgpu_ttm_tt_is_readonly(ttm)) - range->default_flags |= HMM_PFN_REQ_WRITE; - - range->hmm_pfns = kvmalloc_array(ttm->num_pages, - sizeof(*range->hmm_pfns), GFP_KERNEL); - if (unlikely(!range->hmm_pfns)) { - r = -ENOMEM; - goto out_free_ranges; - } - mmap_read_lock(mm); vma = find_vma(mm, start); + mmap_read_unlock(mm); if (unlikely(!vma || start < vma->vm_start)) { r = -EFAULT; - goto out_unlock; + goto out_putmm; } if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) && vma->vm_file)) { r = -EPERM; - goto out_unlock; + goto out_putmm; } - mmap_read_unlock(mm); - timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); - -retry: - range->notifier_seq = mmu_interval_read_begin(&bo->notifier);

- mmap_read_lock(mm); - r = hmm_range_fault(range); - mmap_read_unlock(mm); - if (unlikely(r)) { - /* - * FIXME: This timeout should encompass the retry from - * mmu_interval_read_retry() as well. - */ - if (r == -EBUSY && !time_after(jiffies, timeout)) - goto retry; - goto out_free_pfns; - } - - /* - * Due to default_flags, all pages are HMM_PFN_VALID or - * hmm_range_fault() fails. FIXME: The pages cannot be touched outside - * the notifier_lock, and mmu_interval_read_retry() must be done first. - */ - for (i = 0; i < ttm->num_pages; i++) - pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]); - - gtt->range = range; + readonly = amdgpu_ttm_tt_is_readonly(ttm); + r = amdgpu_hmm_range_get_pages(&bo->notifier, mm, pages, start, + ttm->num_pages, &gtt->range, readonly, + false); +out_putmm: mmput(mm);

- return 0; - -out_unlock: - mmap_read_unlock(mm); -out_free_pfns: - kvfree(range->hmm_pfns); -out_free_ranges: - kfree(range); -out: - mmput(mm); return r; }

@@ -960,10 +907,7 @@ bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm) * FIXME: Must always hold notifier_lock for this, and must * not ignore the return code. */ - r = mmu_interval_read_retry(gtt->range->notifier, - gtt->range->notifier_seq); - kvfree(gtt->range->hmm_pfns); - kfree(gtt->range); + r = amdgpu_hmm_range_get_pages_done(gtt->range); gtt->range = NULL; }

From: Philip Yang Philip.Yang@amd.com

Use HMM to get system memory pages address, which will be used to map to GPUs or migrate to vram.

Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 1 + drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 88 +++++++++++++++++++++++++++ drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 2 + 3 files changed, 91 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index cbb2bae1982d..97cf267b6f51 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -735,6 +735,7 @@ struct svm_range_list { struct work_struct srcu_free_work; struct list_head free_list; struct mutex free_list_lock; + struct mmu_interval_notifier notifier; };

/* Process data */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 017e77e9ae1e..02918faa70d5 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -135,6 +135,65 @@ svm_get_supported_dev_by_id(struct kfd_process *p, uint32_t gpu_id, return dev; }

+/** + * svm_range_validate_ram - get system memory pages of svm range + * + * @mm: the mm_struct of process + * @prange: the range struct + * + * After mapping system memory to GPU, system memory maybe invalidated anytime + * during application running, we use HMM callback to sync GPU with CPU page + * table update, so we don't need use lock to prevent CPU invalidation and check + * hmm_range_get_pages_done return value. + * + * Return: + * 0 - OK, otherwise error code + */ +static int +svm_range_validate_ram(struct mm_struct *mm, struct svm_range *prange) +{ + uint64_t i; + int r; + + if (!prange->pages_addr) { + prange->pages_addr = kvmalloc_array(prange->npages, + sizeof(*prange->pages_addr), + GFP_KERNEL | __GFP_ZERO); + if (!prange->pages_addr) + return -ENOMEM; + } + + r = amdgpu_hmm_range_get_pages(&prange->svms->notifier, mm, NULL, + prange->it_node.start << PAGE_SHIFT, + prange->npages, &prange->hmm_range, + false, true); + if (r) { + pr_debug("failed %d to get svm range pages\n", r); + return r; + } + + for (i = 0; i < prange->npages; i++) + prange->pages_addr[i] = + PFN_PHYS(prange->hmm_range->hmm_pfns[i]); + + amdgpu_hmm_range_get_pages_done(prange->hmm_range); + prange->hmm_range = NULL; + + return 0; +} + +static int +svm_range_validate(struct mm_struct *mm, struct svm_range *prange) +{ + int r = 0; + + pr_debug("actual loc 0x%x\n", prange->actual_loc); + + r = svm_range_validate_ram(mm, prange); + + return r; +} + static int svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange, uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs) @@ -349,10 +408,28 @@ static void svm_range_srcu_free_work(struct work_struct *work_struct) mutex_unlock(&svms->free_list_lock); }

+/** + * svm_range_cpu_invalidate_pagetables - interval notifier callback + * + */ +static bool +svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, + const struct mmu_notifier_range *range, + unsigned long cur_seq) +{ + return true; +} + +static const struct mmu_interval_notifier_ops svm_range_mn_ops = { + .invalidate = svm_range_cpu_invalidate_pagetables, +}; + void svm_range_list_fini(struct kfd_process *p) { pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms);

+ mmu_interval_notifier_remove(&p->svms.notifier); + /* Ensure srcu free work is finished before process is destroyed */ flush_work(&p->svms.srcu_free_work); cleanup_srcu_struct(&p->svms.srcu); @@ -375,6 +452,8 @@ int svm_range_list_init(struct kfd_process *p) INIT_WORK(&svms->srcu_free_work, svm_range_srcu_free_work); INIT_LIST_HEAD(&svms->free_list); mutex_init(&svms->free_list_lock); + mmu_interval_notifier_insert(&svms->notifier, current->mm, 0, ~1ULL, + &svm_range_mn_ops);

return 0; } @@ -531,6 +610,15 @@ svm_range_set_attr(struct kfd_process *p, uint64_t start, uint64_t size, r = svm_range_apply_attrs(p, prange, nattr, attrs); if (r) { pr_debug("failed %d to apply attrs\n", r); + goto out_unlock; + } + + r = svm_range_validate(mm, prange); + if (r) + pr_debug("failed %d to validate svm range\n", r); + +out_unlock: + if (r) { mmap_read_unlock(mm); srcu_read_unlock(&prange->svms->srcu, srcu_idx); goto out_remove; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index c7c54fb73dfb..4d394f72eefc 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -41,6 +41,7 @@ * @list: link list node, used to scan all ranges of svms * @update_list:link list node used to add to update_list * @remove_list:link list node used to add to remove list + * @hmm_range: hmm range structure used by hmm_range_fault to get system pages * @npages: number of pages * @pages_addr: list of system memory physical page address * @flags: flags defined as KFD_IOCTL_SVM_FLAG_* @@ -61,6 +62,7 @@ struct svm_range { struct list_head list; struct list_head update_list; struct list_head remove_list; + struct hmm_range *hmm_range; uint64_t npages; dma_addr_t *pages_addr; uint32_t flags;

From: Philip Yang Philip.Yang@amd.com

When application explicitly call unmap or unmap from mmput when application exit, driver will receive MMU_NOTIFY_UNMAP event to remove svm range from process svms object tree and list first, unmap from GPUs (in the following patch).

Split the svm ranges to handle unmap partial svm range.

Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 86 ++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index ad007261f54c..55500ec4972f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -699,15 +699,101 @@ static void svm_range_srcu_free_work(struct work_struct *work_struct) mutex_unlock(&svms->free_list_lock); }

+static void +svm_range_unmap_from_cpu(struct mm_struct *mm, unsigned long start, + unsigned long last) +{ + struct list_head remove_list; + struct list_head update_list; + struct list_head insert_list; + struct svm_range_list *svms; + struct svm_range new = {0}; + struct svm_range *prange; + struct svm_range *tmp; + struct kfd_process *p; + int r; + + p = kfd_lookup_process_by_mm(mm); + if (!p) + return; + svms = &p->svms; + + pr_debug("notifier svms 0x%p [0x%lx 0x%lx]\n", svms, start, last); + + svms_lock(svms); + + r = svm_range_handle_overlap(svms, &new, start, last, &update_list, + &insert_list, &remove_list, NULL); + if (r) { + svms_unlock(svms); + kfd_unref_process(p); + return; + } + + mutex_lock(&svms->free_list_lock); + list_for_each_entry_safe(prange, tmp, &remove_list, remove_list) { + pr_debug("remove svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + svm_range_unlink(prange); + + pr_debug("schedule to free svms 0x%p [0x%lx 0x%lx]\n", + prange->svms, prange->it_node.start, + prange->it_node.last); + list_add_tail(&prange->remove_list, &svms->free_list); + } + if (!list_empty(&svms->free_list)) + schedule_work(&svms->srcu_free_work); + mutex_unlock(&svms->free_list_lock); + + /* prange in update_list is unmapping from cpu, remove it from insert + * list + */ + list_for_each_entry_safe(prange, tmp, &update_list, update_list) { + list_del(&prange->list); + mutex_lock(&svms->free_list_lock); + list_add_tail(&prange->remove_list, &svms->free_list); + mutex_unlock(&svms->free_list_lock); + } + mutex_lock(&svms->free_list_lock); + if (!list_empty(&svms->free_list)) + schedule_work(&svms->srcu_free_work); + mutex_unlock(&svms->free_list_lock); + + list_for_each_entry_safe(prange, tmp, &insert_list, list) + svm_range_add_to_svms(prange); + + svms_unlock(svms); + kfd_unref_process(p); +} + /** * svm_range_cpu_invalidate_pagetables - interval notifier callback * + * MMU range unmap notifier to remove svm ranges */ static bool svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq) { + unsigned long start = range->start >> PAGE_SHIFT; + unsigned long last = (range->end - 1) >> PAGE_SHIFT; + struct svm_range_list *svms; + + svms = container_of(mni, struct svm_range_list, notifier); + + if (range->event == MMU_NOTIFY_RELEASE) { + pr_debug("cpu release range [0x%lx 0x%lx]\n", range->start, + range->end - 1); + return true; + } + if (range->event == MMU_NOTIFY_UNMAP) { + pr_debug("mm 0x%p unmap range [0x%lx 0x%lx]\n", range->mm, + start, last); + svm_range_unmap_from_cpu(mni->mm, start, last); + return true; + } + return true; }

From: Philip Yang Philip.Yang@amd.com

HMM interval notifier callback notify CPU page table will be updated, stop process queues if the updated address belongs to svm range registered in process svms objects tree. Scheduled restore work to update GPU page table using new pages address in the updated svm range.

svm restore work to use srcu to scan svms list to avoid deadlock between below two cases:

case1: svm restore work takes svm lock to scan svms list, then call hmm_page_fault which takes mm->mmap_sem. case2: unmap event callback and set_attr ioctl takes mm->mmap_sem, than takes svm lock to add/remove ranges.

Calling synchronize_srcu in unmap event callback will deadlock with restore work because restore work may wait for unmap event done to take mm->mmap_sem, so schedule srcu_free_work to wait for srcu read critical section done in svm restore work then free svm ranges.

Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 2 + drivers/gpu/drm/amd/amdkfd/kfd_process.c | 1 + drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 169 ++++++++++++++++++++++- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 2 + 4 files changed, 169 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 97cf267b6f51..f1e95773e19b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -736,6 +736,8 @@ struct svm_range_list { struct list_head free_list; struct mutex free_list_lock; struct mmu_interval_notifier notifier; + atomic_t evicted_ranges; + struct delayed_work restore_work; };

/* Process data */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 791f17308b1b..0f31538b2a91 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -1048,6 +1048,7 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,

cancel_delayed_work_sync(&p->eviction_work); cancel_delayed_work_sync(&p->restore_work); + cancel_delayed_work_sync(&p->svms.restore_work);

mutex_lock(&p->mutex);

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 3c4a036609c4..e3ba6e7262a7 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -21,6 +21,7 @@ */

#include <linux/types.h> +#include <linux/sched/task.h> #include "amdgpu_sync.h" #include "amdgpu_object.h" #include "amdgpu_vm.h" @@ -28,6 +29,8 @@ #include "kfd_priv.h" #include "kfd_svm.h"

+#define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1 + /** * svm_range_unlink - unlink svm_range from lists and interval tree * @prange: svm range structure to be removed @@ -99,6 +102,7 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start, INIT_LIST_HEAD(&prange->list); INIT_LIST_HEAD(&prange->update_list); INIT_LIST_HEAD(&prange->remove_list); + atomic_set(&prange->invalid, 0); svm_range_set_default_attributes(&prange->preferred_loc, &prange->prefetch_loc, &prange->granularity, &prange->flags); @@ -191,6 +195,10 @@ svm_range_validate(struct mm_struct *mm, struct svm_range *prange)

r = svm_range_validate_ram(mm, prange);

+ pr_debug("svms 0x%p [0x%lx 0x%lx] ret %d invalid %d\n", prange->svms, + prange->it_node.start, prange->it_node.last, + r, atomic_read(&prange->invalid)); + return r; }

@@ -757,6 +765,151 @@ static int svm_range_map_to_gpus(struct svm_range *prange, bool reserve_vm) return r; }

+static void svm_range_restore_work(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct amdkfd_process_info *process_info; + struct svm_range_list *svms; + struct svm_range *prange; + struct kfd_process *p; + struct mm_struct *mm; + int evicted_ranges; + int srcu_idx; + int invalid; + int r; + + svms = container_of(dwork, struct svm_range_list, restore_work); + evicted_ranges = atomic_read(&svms->evicted_ranges); + if (!evicted_ranges) + return; + + pr_debug("restore svm ranges\n"); + + /* kfd_process_notifier_release destroys this worker thread. So during + * the lifetime of this thread, kfd_process and mm will be valid. + */ + p = container_of(svms, struct kfd_process, svms); + process_info = p->kgd_process_info; + mm = p->mm; + if (!mm) + return; + + mutex_lock(&process_info->lock); + mmap_read_lock(mm); + srcu_idx = srcu_read_lock(&svms->srcu); + + list_for_each_entry_rcu(prange, &svms->list, list) { + invalid = atomic_read(&prange->invalid); + if (!invalid) + continue; + + pr_debug("restoring svms 0x%p [0x%lx %lx] invalid %d\n", + prange->svms, prange->it_node.start, + prange->it_node.last, invalid); + + r = svm_range_validate(mm, prange); + if (r) { + pr_debug("failed %d to validate [0x%lx 0x%lx]\n", r, + prange->it_node.start, prange->it_node.last); + + goto unlock_out; + } + + r = svm_range_map_to_gpus(prange, true); + if (r) { + pr_debug("failed %d to map 0x%lx to gpu\n", r, + prange->it_node.start); + goto unlock_out; + } + + if (atomic_cmpxchg(&prange->invalid, invalid, 0) != invalid) + goto unlock_out; + } + + if (atomic_cmpxchg(&svms->evicted_ranges, evicted_ranges, 0) != + evicted_ranges) + goto unlock_out; + + evicted_ranges = 0; + + r = kgd2kfd_resume_mm(mm); + if (r) { + /* No recovery from this failure. Probably the CP is + * hanging. No point trying again. + */ + pr_debug("failed %d to resume KFD\n", r); + } + + pr_debug("restore svm ranges successfully\n"); + +unlock_out: + srcu_read_unlock(&svms->srcu, srcu_idx); + mmap_read_unlock(mm); + mutex_unlock(&process_info->lock); + + /* If validation failed, reschedule another attempt */ + if (evicted_ranges) { + pr_debug("reschedule to restore svm range\n"); + schedule_delayed_work(&svms->restore_work, + msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS)); + } +} + +/** + * svm_range_evict - evict svm range + * + * Stop all queues of the process to ensure GPU doesn't access the memory, then + * return to let CPU evict the buffer and proceed CPU pagetable update. + * + * Don't need use lock to sync cpu pagetable invalidation with GPU execution. + * If invalidation happens while restore work is running, restore work will + * restart to ensure to get the latest CPU pages mapping to GPU, then start + * the queues. + */ +static int +svm_range_evict(struct svm_range_list *svms, struct mm_struct *mm, + unsigned long start, unsigned long last) +{ + int invalid, evicted_ranges; + int r = 0; + struct interval_tree_node *node; + struct svm_range *prange; + + svms_lock(svms); + + pr_debug("invalidate svms 0x%p [0x%lx 0x%lx]\n", svms, start, last); + + node = interval_tree_iter_first(&svms->objects, start, last); + while (node) { + struct interval_tree_node *next; + + prange = container_of(node, struct svm_range, it_node); + next = interval_tree_iter_next(node, start, last); + + invalid = atomic_inc_return(&prange->invalid); + evicted_ranges = atomic_inc_return(&svms->evicted_ranges); + if (evicted_ranges == 1) { + pr_debug("evicting svms 0x%p range [0x%lx 0x%lx]\n", + prange->svms, prange->it_node.start, + prange->it_node.last); + + /* First eviction, stop the queues */ + r = kgd2kfd_quiesce_mm(mm); + if (r) + pr_debug("failed to quiesce KFD\n"); + + pr_debug("schedule to restore svm %p ranges\n", svms); + schedule_delayed_work(&svms->restore_work, + msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS)); + } + node = next; + } + + svms_unlock(svms); + + return r; +} + struct svm_range *svm_range_clone(struct svm_range *old) { struct svm_range *new; @@ -994,6 +1147,11 @@ svm_range_unmap_from_cpu(struct mm_struct *mm, unsigned long start, * svm_range_cpu_invalidate_pagetables - interval notifier callback * * MMU range unmap notifier to remove svm ranges + * + * If GPU vm fault retry is not enabled, evict the svm range, then restore + * work will update GPU mapping. + * If GPU vm fault retry is enabled, unmap the svm range from GPU, vm fault + * will update GPU mapping. */ static bool svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, @@ -1009,15 +1167,14 @@ svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, if (range->event == MMU_NOTIFY_RELEASE) { pr_debug("cpu release range [0x%lx 0x%lx]\n", range->start, range->end - 1); - return true; - } - if (range->event == MMU_NOTIFY_UNMAP) { + } else if (range->event == MMU_NOTIFY_UNMAP) { pr_debug("mm 0x%p unmap range [0x%lx 0x%lx]\n", range->mm, start, last); svm_range_unmap_from_cpu(mni->mm, start, last); - return true; + } else { + mmu_interval_set_seq(mni, cur_seq); + svm_range_evict(svms, mni->mm, start, last); } - return true; }

@@ -1045,6 +1202,8 @@ int svm_range_list_init(struct kfd_process *p) svms->objects = RB_ROOT_CACHED; mutex_init(&svms->lock); INIT_LIST_HEAD(&svms->list); + atomic_set(&svms->evicted_ranges, 0); + INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work); r = init_srcu_struct(&svms->srcu); if (r) { pr_debug("failed %d to init srcu\n", r); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index fb68b5ee54f8..4c7daf8e0b6f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -50,6 +50,7 @@ * @perfetch_loc: last prefetch location, 0 for CPU, or GPU id * @actual_loc: the actual location, 0 for CPU, or GPU id * @granularity:migration granularity, log2 num pages + * @invalid: not 0 means cpu page table is invalidated * @bitmap_access: index bitmap of GPUs which can access the range * @bitmap_aip: index bitmap of GPUs which can access the range in place * @@ -72,6 +73,7 @@ struct svm_range { uint32_t prefetch_loc; uint32_t actual_loc; uint8_t granularity; + atomic_t invalid; DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE); DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE); };

From: Alex Sierra alex.sierra@amd.com

Xnack retries are used for page fault recovery. Some AMD chip families support continuously retry while page table entries are invalid. The driver must handle the page fault interrupt and fill in a valid entry for the GPU to continue.

This ioctl allows to enable/disable XNACK retries per KFD process.

Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 28 +++++++++++++++ include/uapi/linux/kfd_ioctl.h | 43 +++++++++++++++++++++++- 2 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 2d3ba7e806d5..a9a6a7c8ff21 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1747,6 +1747,31 @@ static int kfd_ioctl_smi_events(struct file *filep, return kfd_smi_event_open(dev, &args->anon_fd); }

+static int kfd_ioctl_set_xnack_mode(struct file *filep, + struct kfd_process *p, void *data) +{ + struct kfd_ioctl_set_xnack_mode_args *args = data; + int r = 0; + + mutex_lock(&p->mutex); + if (args->xnack_enabled >= 0) { + if (!list_empty(&p->pqm.queues)) { + pr_debug("Process has user queues running\n"); + mutex_unlock(&p->mutex); + return -EBUSY; + } + if (args->xnack_enabled && !kfd_process_xnack_supported(p)) + r = -EPERM; + else + p->xnack_enabled = args->xnack_enabled; + } else { + args->xnack_enabled = p->xnack_enabled; + } + mutex_unlock(&p->mutex); + + return r; +} + static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data) { struct kfd_ioctl_svm_args *args = data; @@ -1870,6 +1895,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { kfd_ioctl_smi_events, 0),

AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0), + + AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE, + kfd_ioctl_set_xnack_mode, 0), };

#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index 5d4a4b3e0b61..b1a45cd37ab7 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -593,6 +593,44 @@ struct kfd_ioctl_svm_args { struct kfd_ioctl_svm_attribute attrs[0]; };

+/** + * kfd_ioctl_set_xnack_mode_args - Arguments for set_xnack_mode + * + * @xnack_enabled: [in/out] Whether to enable XNACK mode for this process + * + * @xnack_enabled indicates whether recoverable page faults should be + * enabled for the current process. 0 means disabled, positive means + * enabled, negative means leave unchanged. If enabled, virtual address + * translations on GFXv9 and later AMD GPUs can return XNACK and retry + * the access until a valid PTE is available. This is used to implement + * device page faults. + * + * On output, @xnack_enabled returns the (new) current mode (0 or + * positive). Therefore, a negative input value can be used to query + * the current mode without changing it. + * + * The XNACK mode fundamentally changes the way SVM managed memory works + * in the driver, with subtle effects on application performance and + * functionality. + * + * Enabling XNACK mode requires shader programs to be compiled + * differently. Furthermore, not all GPUs support changing the mode + * per-process. Therefore changing the mode is only allowed while no + * user mode queues exist in the process. This ensure that no shader + * code is running that may be compiled for the wrong mode. And GPUs + * that cannot change to the requested mode will prevent the XNACK + * mode from occurring. All GPUs used by the process must be in the + * same XNACK mode. + * + * GFXv8 or older GPUs do not support 48 bit virtual addresses or SVM. + * Therefore those GPUs are not considered for the XNACK mode switch. + * + * Return: 0 on success, -errno on failure + */ +struct kfd_ioctl_set_xnack_mode_args { + __s32 xnack_enabled; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -695,7 +733,10 @@ struct kfd_ioctl_svm_args {

#define AMDKFD_IOC_SVM AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args)

+#define AMDKFD_IOC_SET_XNACK_MODE \ + AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x21 +#define AMDKFD_COMMAND_END 0x22

#endif

From: Philip Yang Philip.Yang@amd.com

If svm range perfetch location is not zero, use TTM to alloc amdgpu_bo vram nodes to validate svm range, then map vram nodes to GPUs.

Use offset to sub allocate from the same amdgpu_bo to handle overlap vram range while adding new range or unmapping range.

svm_bo has ref count to trace the shared ranges. If all ranges of shared amdgpu_bo are migrated to ram, ref count becomes 0, then amdgpu_bo is released, all ranges svm_bo is set to NULL.

To migrate range from ram back to vram, allocate the same amdgpu_bo with previous offset if the range has svm_bo.

Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 342 ++++++++++++++++++++++++--- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 20 ++ 2 files changed, 335 insertions(+), 27 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index e3ba6e7262a7..7d91dc49a5a9 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -35,7 +35,9 @@ * svm_range_unlink - unlink svm_range from lists and interval tree * @prange: svm range structure to be removed * - * Remove the svm range from svms interval tree and link list + * Remove the svm_range from the svms and svm_bo SRCU lists and the svms + * interval tree. After this call, synchronize_srcu is needed before the + * range can be freed safely. * * Context: The caller must hold svms_lock */ @@ -44,6 +46,12 @@ static void svm_range_unlink(struct svm_range *prange) pr_debug("prange 0x%p [0x%lx 0x%lx]\n", prange, prange->it_node.start, prange->it_node.last);

+ if (prange->svm_bo) { + spin_lock(&prange->svm_bo->list_lock); + list_del(&prange->svm_bo_list); + spin_unlock(&prange->svm_bo->list_lock); + } + list_del_rcu(&prange->list); interval_tree_remove(&prange->it_node, &prange->svms->objects); } @@ -70,6 +78,12 @@ static void svm_range_remove(struct svm_range *prange) pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->it_node.start, prange->it_node.last);

+ if (prange->mm_nodes) { + pr_debug("vram prange svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + svm_range_vram_node_free(prange); + } + kvfree(prange->pages_addr); kfree(prange); } @@ -102,7 +116,9 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start, INIT_LIST_HEAD(&prange->list); INIT_LIST_HEAD(&prange->update_list); INIT_LIST_HEAD(&prange->remove_list); + INIT_LIST_HEAD(&prange->svm_bo_list); atomic_set(&prange->invalid, 0); + spin_lock_init(&prange->svm_bo_lock); svm_range_set_default_attributes(&prange->preferred_loc, &prange->prefetch_loc, &prange->granularity, &prange->flags); @@ -139,6 +155,16 @@ svm_get_supported_dev_by_id(struct kfd_process *p, uint32_t gpu_id, return dev; }

+struct amdgpu_device * +svm_range_get_adev_by_id(struct svm_range *prange, uint32_t gpu_id) +{ + struct kfd_process *p = + container_of(prange->svms, struct kfd_process, svms); + struct kfd_dev *dev = svm_get_supported_dev_by_id(p, gpu_id, NULL); + + return dev ? (struct amdgpu_device *)dev->kgd : NULL; +} + /** * svm_range_validate_ram - get system memory pages of svm range * @@ -186,14 +212,226 @@ svm_range_validate_ram(struct mm_struct *mm, struct svm_range *prange) return 0; }

+static bool svm_bo_ref_unless_zero(struct svm_range_bo *svm_bo) +{ + if (!svm_bo || !kref_get_unless_zero(&svm_bo->kref)) + return false; + + return true; +} + +static struct svm_range_bo *svm_range_bo_ref(struct svm_range_bo *svm_bo) +{ + if (svm_bo) + kref_get(&svm_bo->kref); + + return svm_bo; +} + +static void svm_range_bo_release(struct kref *kref) +{ + struct svm_range_bo *svm_bo; + + svm_bo = container_of(kref, struct svm_range_bo, kref); + /* This cleanup loop does not need to be SRCU safe because there + * should be no SRCU readers while the ref count is 0. Any SRCU + * reader that has a chance of reducing the ref count must take + * an extra reference before srcu_read_lock and release it after + * srcu_read_unlock. + */ + spin_lock(&svm_bo->list_lock); + while (!list_empty(&svm_bo->range_list)) { + struct svm_range *prange = + list_first_entry(&svm_bo->range_list, + struct svm_range, svm_bo_list); + pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + spin_lock(&prange->svm_bo_lock); + prange->svm_bo = NULL; + spin_unlock(&prange->svm_bo_lock); + + /* list_del_init tells a concurrent svm_range_vram_node_new when + * it's safe to reuse the svm_bo pointer and svm_bo_list head. + */ + list_del_init(&prange->svm_bo_list); + } + spin_unlock(&svm_bo->list_lock); + + amdgpu_bo_unref(&svm_bo->bo); + kfree(svm_bo); +} + +static void svm_range_bo_unref(struct svm_range_bo *svm_bo) +{ + if (!svm_bo) + return; + + kref_put(&svm_bo->kref, svm_range_bo_release); +} + +static struct svm_range_bo *svm_range_bo_new(void) +{ + struct svm_range_bo *svm_bo; + + svm_bo = kzalloc(sizeof(*svm_bo), GFP_KERNEL); + if (!svm_bo) + return NULL; + + kref_init(&svm_bo->kref); + INIT_LIST_HEAD(&svm_bo->range_list); + spin_lock_init(&svm_bo->list_lock); + + return svm_bo; +} + +int +svm_range_vram_node_new(struct amdgpu_device *adev, struct svm_range *prange, + bool clear) +{ + struct amdkfd_process_info *process_info; + struct amdgpu_bo_param bp; + struct svm_range_bo *svm_bo; + struct amdgpu_bo *bo; + struct kfd_process *p; + int r; + + pr_debug("[0x%lx 0x%lx]\n", prange->it_node.start, + prange->it_node.last); + spin_lock(&prange->svm_bo_lock); + if (prange->svm_bo) { + if (prange->mm_nodes) { + /* We still have a reference, all is well */ + spin_unlock(&prange->svm_bo_lock); + return 0; + } + if (svm_bo_ref_unless_zero(prange->svm_bo)) { + /* The BO was still around and we got + * a new reference to it + */ + spin_unlock(&prange->svm_bo_lock); + pr_debug("reuse old bo [0x%lx 0x%lx]\n", + prange->it_node.start, prange->it_node.last); + + prange->mm_nodes = prange->svm_bo->bo->tbo.mem.mm_node; + return 0; + } + + spin_unlock(&prange->svm_bo_lock); + + /* We need a new svm_bo. Spin-loop to wait for concurrent + * svm_range_bo_release to finish removing this range from + * its range list. After this, it is safe to reuse the + * svm_bo pointer and svm_bo_list head. + */ + while (!list_empty_careful(&prange->svm_bo_list)) + ; + + } else { + spin_unlock(&prange->svm_bo_lock); + } + + svm_bo = svm_range_bo_new(); + if (!svm_bo) { + pr_debug("failed to alloc svm bo\n"); + return -ENOMEM; + } + + memset(&bp, 0, sizeof(bp)); + bp.size = prange->npages * PAGE_SIZE; + bp.byte_align = PAGE_SIZE; + bp.domain = AMDGPU_GEM_DOMAIN_VRAM; + bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS; + bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0; + bp.type = ttm_bo_type_device; + bp.resv = NULL; + + r = amdgpu_bo_create(adev, &bp, &bo); + if (r) { + pr_debug("failed %d to create bo\n", r); + kfree(svm_bo); + return r; + } + + p = container_of(prange->svms, struct kfd_process, svms); + r = amdgpu_bo_reserve(bo, true); + if (r) { + pr_debug("failed %d to reserve bo\n", r); + goto reserve_bo_failed; + } + + r = dma_resv_reserve_shared(bo->tbo.base.resv, 1); + if (r) { + pr_debug("failed %d to reserve bo\n", r); + amdgpu_bo_unreserve(bo); + goto reserve_bo_failed; + } + process_info = p->kgd_process_info; + amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true); + + amdgpu_bo_unreserve(bo); + + svm_bo->bo = bo; + prange->svm_bo = svm_bo; + prange->mm_nodes = bo->tbo.mem.mm_node; + prange->offset = 0; + + spin_lock(&svm_bo->list_lock); + list_add(&prange->svm_bo_list, &svm_bo->range_list); + spin_unlock(&svm_bo->list_lock); + + return 0; + +reserve_bo_failed: + kfree(svm_bo); + amdgpu_bo_unref(&bo); + prange->mm_nodes = NULL; + + return r; +} + +void svm_range_vram_node_free(struct svm_range *prange) +{ + pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + + svm_range_bo_unref(prange->svm_bo); + prange->mm_nodes = NULL; +} + +static int svm_range_validate_vram(struct svm_range *prange) +{ + struct amdgpu_device *adev; + int r; + + pr_debug("svms 0x%p [0x%lx 0x%lx] actual_loc 0x%x\n", prange->svms, + prange->it_node.start, prange->it_node.last, + prange->actual_loc); + + adev = svm_range_get_adev_by_id(prange, prange->actual_loc); + if (!adev) { + pr_debug("failed to get device by id 0x%x\n", + prange->actual_loc); + return -EINVAL; + } + + r = svm_range_vram_node_new(adev, prange, true); + if (r) + pr_debug("failed %d to alloc vram\n", r); + + return r; +} + static int svm_range_validate(struct mm_struct *mm, struct svm_range *prange) { - int r = 0; + int r;

pr_debug("actual loc 0x%x\n", prange->actual_loc);

- r = svm_range_validate_ram(mm, prange); + if (!prange->actual_loc) + r = svm_range_validate_ram(mm, prange); + else + r = svm_range_validate_vram(prange);

pr_debug("svms 0x%p [0x%lx 0x%lx] ret %d invalid %d\n", prange->svms, prange->it_node.start, prange->it_node.last, @@ -349,6 +587,35 @@ svm_range_split_pages(struct svm_range *new, struct svm_range *old, return 0; }

+static int +svm_range_split_nodes(struct svm_range *new, struct svm_range *old, + uint64_t start, uint64_t last) +{ + pr_debug("svms 0x%p new start 0x%lx start 0x%llx last 0x%llx\n", + new->svms, new->it_node.start, start, last); + + old->npages = last - start + 1; + + if (new->it_node.start == old->it_node.start) { + new->offset = old->offset; + old->offset += new->npages; + } else { + new->offset = old->offset + old->npages; + } + + old->it_node.start = start; + old->it_node.last = last; + + new->svm_bo = svm_range_bo_ref(old->svm_bo); + new->mm_nodes = old->mm_nodes; + + spin_lock(&new->svm_bo->list_lock); + list_add(&new->svm_bo_list, &new->svm_bo->range_list); + spin_unlock(&new->svm_bo->list_lock); + + return 0; +} + /** * svm_range_split_adjust - split range and adjust * @@ -382,6 +649,8 @@ svm_range_split_adjust(struct svm_range *new, struct svm_range *old,

if (old->pages_addr) r = svm_range_split_pages(new, old, start, last); + else if (old->actual_loc && old->mm_nodes) + r = svm_range_split_nodes(new, old, start, last); else WARN_ONCE(1, "split adjust invalid pages_addr and nodes\n"); if (r) @@ -438,17 +707,14 @@ svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last, return -EINVAL;

svms = prange->svms; - if (old_start == start) { + if (old_start == start) *new = svm_range_new(svms, last + 1, old_last); - if (!*new) - return -ENOMEM; - r = svm_range_split_adjust(*new, prange, start, last); - } else { + else *new = svm_range_new(svms, old_start, start - 1); - if (!*new) - return -ENOMEM; - r = svm_range_split_adjust(*new, prange, start, last); - } + if (!*new) + return -ENOMEM; + + r = svm_range_split_adjust(*new, prange, start, last);

return r; } @@ -550,7 +816,8 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange) uint64_t pte_flags;

pte_flags = AMDGPU_PTE_VALID; - pte_flags |= AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED; + if (!prange->mm_nodes) + pte_flags |= AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED;

mapping_flags = AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE;

@@ -570,7 +837,9 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange) /* Apply ASIC specific mapping flags */ amdgpu_gmc_get_vm_pte(adev, &prange->mapping, &pte_flags);

- pr_debug("PTE flags 0x%llx\n", pte_flags); + pr_debug("svms 0x%p [0x%lx 0x%lx] vram %d system %d PTE flags 0x%llx\n", + prange->svms, prange->it_node.start, prange->it_node.last, + prange->mm_nodes ? 1:0, prange->pages_addr ? 1:0, pte_flags);

return pte_flags; } @@ -656,7 +925,9 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct svm_range *prange, bool reserve_vm, struct dma_fence **fence) { - struct amdgpu_bo *root; + struct ttm_validate_buffer tv[2]; + struct ww_acquire_ctx ticket; + struct list_head list; dma_addr_t *pages_addr; uint64_t pte_flags; int r = 0; @@ -665,13 +936,25 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, prange->it_node.start, prange->it_node.last);

if (reserve_vm) { - root = amdgpu_bo_ref(vm->root.base.bo); - r = amdgpu_bo_reserve(root, true); + INIT_LIST_HEAD(&list); + + tv[0].bo = &vm->root.base.bo->tbo; + tv[0].num_shared = 4; + list_add(&tv[0].head, &list); + if (prange->svm_bo && prange->mm_nodes) { + tv[1].bo = &prange->svm_bo->bo->tbo; + tv[1].num_shared = 1; + list_add(&tv[1].head, &list); + } + r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL); if (r) { - pr_debug("failed %d to reserve root bo\n", r); - amdgpu_bo_unref(&root); + pr_debug("failed %d to reserve bo\n", r); goto out; } + if (prange->svm_bo && prange->mm_nodes && + prange->svm_bo->bo->tbo.evicted) + goto unreserve_out; + r = amdgpu_vm_validate_pt_bos(adev, vm, svm_range_bo_validate, NULL); if (r) { @@ -682,7 +965,7 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,

prange->mapping.start = prange->it_node.start; prange->mapping.last = prange->it_node.last; - prange->mapping.offset = 0; + prange->mapping.offset = prange->offset; pte_flags = svm_range_get_pte_flags(adev, prange); prange->mapping.flags = pte_flags; pages_addr = prange->pages_addr; @@ -690,7 +973,8 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, r = amdgpu_vm_bo_update_mapping(adev, adev, vm, false, false, NULL, prange->mapping.start, prange->mapping.last, pte_flags, - prange->mapping.offset, NULL, + prange->mapping.offset, + prange->mm_nodes, pages_addr, &vm->last_update); if (r) { pr_debug("failed %d to map to gpu 0x%lx\n", r, @@ -710,11 +994,8 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, *fence = dma_fence_get(vm->last_update);

unreserve_out: - if (reserve_vm) { - amdgpu_bo_unreserve(root); - amdgpu_bo_unref(&root); - } - + if (reserve_vm) + ttm_eu_backoff_reservation(&ticket, &list); out: return r; } @@ -929,7 +1210,14 @@ struct svm_range *svm_range_clone(struct svm_range *old) memcpy(new->pages_addr, old->pages_addr, old->npages * sizeof(*old->pages_addr)); } - + if (old->svm_bo) { + new->mm_nodes = old->mm_nodes; + new->offset = old->offset; + new->svm_bo = svm_range_bo_ref(old->svm_bo); + spin_lock(&new->svm_bo->list_lock); + list_add(&new->svm_bo_list, &new->svm_bo->range_list); + spin_unlock(&new->svm_bo->list_lock); + } new->flags = old->flags; new->preferred_loc = old->preferred_loc; new->prefetch_loc = old->prefetch_loc; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index 4c7daf8e0b6f..b1d2db02043b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -32,6 +32,12 @@ #include "amdgpu.h" #include "kfd_priv.h"

+struct svm_range_bo { + struct amdgpu_bo *bo; + struct kref kref; + struct list_head range_list; /* all svm ranges shared this bo */ + spinlock_t list_lock; +}; /** * struct svm_range - shared virtual memory range * @@ -45,6 +51,10 @@ * @mapping: bo_va mapping structure to create and update GPU page table * @npages: number of pages * @pages_addr: list of system memory physical page address + * @mm_nodes: vram nodes allocated + * @offset: range start offset within mm_nodes + * @svm_bo: struct to manage splited amdgpu_bo + * @svm_bo_list:link list node, to scan all ranges which share same svm_bo * @flags: flags defined as KFD_IOCTL_SVM_FLAG_* * @perferred_loc: perferred location, 0 for CPU, or GPU id * @perfetch_loc: last prefetch location, 0 for CPU, or GPU id @@ -68,6 +78,11 @@ struct svm_range { struct amdgpu_bo_va_mapping mapping; uint64_t npages; dma_addr_t *pages_addr; + struct drm_mm_node *mm_nodes; + uint64_t offset; + struct svm_range_bo *svm_bo; + struct list_head svm_bo_list; + spinlock_t svm_bo_lock; uint32_t flags; uint32_t preferred_loc; uint32_t prefetch_loc; @@ -95,5 +110,10 @@ void svm_range_list_fini(struct kfd_process *p); int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start, uint64_t size, uint32_t nattrs, struct kfd_ioctl_svm_attribute *attrs); +struct amdgpu_device *svm_range_get_adev_by_id(struct svm_range *prange, + uint32_t id); +int svm_range_vram_node_new(struct amdgpu_device *adev, + struct svm_range *prange, bool clear); +void svm_range_vram_node_free(struct svm_range *prange);

#endif /* KFD_SVM_H_ */

From: Philip Yang Philip.Yang@amd.com

Use sdma linear copy to migrate data between ram and vram. The sdma linear copy command uses kernel buffer function queue to access system memory through gart table.

Use reserved gart table window 0 to map system page address, and vram page address is direct mapping. Use the same kernel buffer function to fill in gart table mapping, so this is serialized with memory copy by sdma job submit. We only need wait for the last memory copy sdma fence for larger buffer migration.

Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 172 +++++++++++++++++++++++ drivers/gpu/drm/amd/amdkfd/kfd_migrate.h | 5 + 2 files changed, 177 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 1950b86f1562..f2019c8f0b80 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -32,6 +32,178 @@ #include "kfd_svm.h" #include "kfd_migrate.h"

+static uint64_t +svm_migrate_direct_mapping_addr(struct amdgpu_device *adev, uint64_t addr) +{ + return addr + amdgpu_ttm_domain_start(adev, TTM_PL_VRAM); +} + +static int +svm_migrate_gart_map(struct amdgpu_ring *ring, uint64_t npages, + uint64_t *addr, uint64_t *gart_addr, uint64_t flags) +{ + struct amdgpu_device *adev = ring->adev; + struct amdgpu_job *job; + unsigned int num_dw, num_bytes; + struct dma_fence *fence; + uint64_t src_addr, dst_addr; + uint64_t pte_flags; + void *cpu_addr; + int r; + + /* use gart window 0 */ + *gart_addr = adev->gmc.gart_start; + + num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); + num_bytes = npages * 8; + + r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, + AMDGPU_IB_POOL_DELAYED, &job); + if (r) + return r; + + src_addr = num_dw * 4; + src_addr += job->ibs[0].gpu_addr; + + dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo); + amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, + dst_addr, num_bytes, false); + + amdgpu_ring_pad_ib(ring, &job->ibs[0]); + WARN_ON(job->ibs[0].length_dw > num_dw); + + pte_flags = AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE; + pte_flags |= AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED; + if (!(flags & KFD_IOCTL_SVM_FLAG_GPU_RO)) + pte_flags |= AMDGPU_PTE_WRITEABLE; + pte_flags |= adev->gart.gart_pte_flags; + + cpu_addr = &job->ibs[0].ptr[num_dw]; + + r = amdgpu_gart_map(adev, 0, npages, addr, pte_flags, cpu_addr); + if (r) + goto error_free; + + r = amdgpu_job_submit(job, &adev->mman.entity, + AMDGPU_FENCE_OWNER_UNDEFINED, &fence); + if (r) + goto error_free; + + dma_fence_put(fence); + + return r; + +error_free: + amdgpu_job_free(job); + return r; +} + +/** + * svm_migrate_copy_memory_gart - sdma copy data between ram and vram + * + * @adev: amdgpu device the sdma ring running + * @src: source page address array + * @dst: destination page address array + * @npages: number of pages to copy + * @direction: enum MIGRATION_COPY_DIR + * @mfence: output, sdma fence to signal after sdma is done + * + * ram address uses GART table continuous entries mapping to ram pages, + * vram address uses direct mapping of vram pages, which must have npages + * number of continuous pages. + * GART update and sdma uses same buf copy function ring, sdma is splited to + * multiple GTT_MAX_PAGES transfer, all sdma operations are serialized, wait for + * the last sdma finish fence which is returned to check copy memory is done. + * + * Context: Process context, takes and releases gtt_window_lock + * + * Return: + * 0 - OK, otherwise error code + */ + +static int +svm_migrate_copy_memory_gart(struct amdgpu_device *adev, uint64_t *src, + uint64_t *dst, uint64_t npages, + enum MIGRATION_COPY_DIR direction, + struct dma_fence **mfence) +{ + const uint64_t GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE; + struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; + uint64_t gart_s, gart_d; + struct dma_fence *next; + uint64_t size; + int r; + + mutex_lock(&adev->mman.gtt_window_lock); + + while (npages) { + size = min(GTT_MAX_PAGES, npages); + + if (direction == FROM_VRAM_TO_RAM) { + gart_s = svm_migrate_direct_mapping_addr(adev, *src); + r = svm_migrate_gart_map(ring, size, dst, &gart_d, 0); + + } else if (direction == FROM_RAM_TO_VRAM) { + r = svm_migrate_gart_map(ring, size, src, &gart_s, + KFD_IOCTL_SVM_FLAG_GPU_RO); + gart_d = svm_migrate_direct_mapping_addr(adev, *dst); + } + if (r) { + pr_debug("failed %d to create gart mapping\n", r); + goto out_unlock; + } + + r = amdgpu_copy_buffer(ring, gart_s, gart_d, size * PAGE_SIZE, + NULL, &next, false, true, false); + if (r) { + pr_debug("failed %d to copy memory\n", r); + goto out_unlock; + } + + dma_fence_put(*mfence); + *mfence = next; + npages -= size; + if (npages) { + src += size; + dst += size; + } + } + +out_unlock: + mutex_unlock(&adev->mman.gtt_window_lock); + + return r; +} + +/** + * svm_migrate_copy_done - wait for memory copy sdma is done + * + * @adev: amdgpu device the sdma memory copy is executing on + * @mfence: migrate fence + * + * Wait for dma fence is signaled, if the copy ssplit into multiple sdma + * operations, this is the last sdma operation fence. + * + * Context: called after svm_migrate_copy_memory + * + * Return: + * 0 - success + * otherwise - error code from dma fence signal + */ +int +svm_migrate_copy_done(struct amdgpu_device *adev, struct dma_fence *mfence) +{ + int r = 0; + + if (mfence) { + r = dma_fence_wait(mfence, false); + dma_fence_put(mfence); + pr_debug("sdma copy memory fence done\n"); + } + + return r; +} + static void svm_migrate_page_free(struct page *page) { } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h index 98ab685d3e17..5db5686fa46a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h @@ -32,6 +32,11 @@ #include "kfd_priv.h" #include "kfd_svm.h"

+enum MIGRATION_COPY_DIR { + FROM_RAM_TO_VRAM = 0, + FROM_VRAM_TO_RAM +}; + #if defined(CONFIG_DEVICE_PRIVATE) int svm_migrate_init(struct amdgpu_device *adev); void svm_migrate_fini(struct amdgpu_device *adev);

From: Philip Yang Philip.Yang@amd.com

If CPU page fault happens, HMM pgmap_ops callback migrate_to_ram start migrate memory from vram to ram in steps:

1. migrate_vma_pages get vram pages, and notify HMM to invalidate the pages, HMM interval notifier callback evict process queues 2. Allocate system memory pages 3. Use svm copy memory to migrate data from vram to ram 4. migrate_vma_pages copy pages structure from vram pages to ram pages 5. Return VM_FAULT_SIGBUS if migration failed, to notify application 6. migrate_vma_finalize put vram pages, page_free callback free vram pages and vram nodes 7. Restore work wait for migration is finished, then update GPU page table mapping to system memory, and resume process queues

Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 274 ++++++++++++++++++++++- drivers/gpu/drm/amd/amdkfd/kfd_migrate.h | 3 + drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 116 +++++++++- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 4 + 4 files changed, 392 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index af23f0be7eaf..d33a4cc63495 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -259,6 +259,35 @@ svm_migrate_put_vram_page(struct amdgpu_device *adev, unsigned long addr) put_page(page); }

+static unsigned long +svm_migrate_addr(struct amdgpu_device *adev, struct page *page) +{ + unsigned long addr; + + addr = page_to_pfn(page) << PAGE_SHIFT; + return (addr - adev->kfd.dev->pgmap.res.start); +} + +static struct page * +svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr) +{ + struct page *page; + + page = alloc_page_vma(GFP_HIGHUSER, vma, addr); + if (page) + lock_page(page); + + return page; +} + +void svm_migrate_put_sys_page(unsigned long addr) +{ + struct page *page; + + page = pfn_to_page(addr >> PAGE_SHIFT); + unlock_page(page); + put_page(page); +}

static int svm_migrate_copy_to_vram(struct amdgpu_device *adev, struct svm_range *prange, @@ -471,13 +500,208 @@ int svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc)

static void svm_migrate_page_free(struct page *page) { + /* Keep this function to avoid warning */ +} + +static int +svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, + struct migrate_vma *migrate, + struct dma_fence **mfence) +{ + uint64_t npages = migrate->cpages; + uint64_t *src, *dst; + struct page *dpage; + uint64_t i = 0, j; + uint64_t addr; + int r = 0; + + pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + + addr = prange->it_node.start << PAGE_SHIFT; + + src = kvmalloc_array(npages << 1, sizeof(*src), GFP_KERNEL); + if (!src) + return -ENOMEM; + + dst = src + npages; + + prange->pages_addr = kvmalloc_array(npages, sizeof(*prange->pages_addr), + GFP_KERNEL | __GFP_ZERO); + if (!prange->pages_addr) { + r = -ENOMEM; + goto out_oom; + } + + for (i = 0, j = 0; i < npages; i++, j++, addr += PAGE_SIZE) { + struct page *spage; + + spage = migrate_pfn_to_page(migrate->src[i]); + if (!spage) { + pr_debug("failed get spage svms 0x%p [0x%lx 0x%lx]\n", + prange->svms, prange->it_node.start, + prange->it_node.last); + r = -ENOMEM; + goto out_oom; + } + src[i] = svm_migrate_addr(adev, spage); + if (i > 0 && src[i] != src[i - 1] + PAGE_SIZE) { + r = svm_migrate_copy_memory_gart(adev, src + i - j, + dst + i - j, j, + FROM_VRAM_TO_RAM, + mfence); + if (r) + goto out_oom; + j = 0; + } + + dpage = svm_migrate_get_sys_page(migrate->vma, addr); + if (!dpage) { + pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n", + prange->svms, prange->it_node.start, + prange->it_node.last); + r = -ENOMEM; + goto out_oom; + } + + dst[i] = page_to_pfn(dpage) << PAGE_SHIFT; + *(prange->pages_addr + i) = dst[i]; + + migrate->dst[i] = migrate_pfn(page_to_pfn(dpage)); + migrate->dst[i] |= MIGRATE_PFN_LOCKED; + + } + + r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j, + FROM_VRAM_TO_RAM, mfence); + +out_oom: + kvfree(src); + if (r) { + pr_debug("failed %d copy to ram\n", r); + while (i--) { + svm_migrate_put_sys_page(dst[i]); + migrate->dst[i] = 0; + } + } + + return r; +} + +static int +svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange, + struct vm_area_struct *vma, uint64_t start, uint64_t end) +{ + uint64_t npages = (end - start) >> PAGE_SHIFT; + struct dma_fence *mfence = NULL; + struct migrate_vma migrate; + int r = -ENOMEM; + + memset(&migrate, 0, sizeof(migrate)); + migrate.vma = vma; + migrate.start = start; + migrate.end = end; + migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE; + migrate.pgmap_owner = adev; + + migrate.src = kvmalloc_array(npages << 1, sizeof(*migrate.src), + GFP_KERNEL | __GFP_ZERO); + if (!migrate.src) + goto out; + migrate.dst = migrate.src + npages; + + r = migrate_vma_setup(&migrate); + if (r) { + pr_debug("failed %d prepare migrate svms 0x%p [0x%lx 0x%lx]\n", + r, prange->svms, prange->it_node.start, + prange->it_node.last); + goto out_free; + } + + pr_debug("cpages %ld\n", migrate.cpages); + + if (migrate.cpages) { + svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence); + migrate_vma_pages(&migrate); + svm_migrate_copy_done(adev, mfence); + migrate_vma_finalize(&migrate); + } else { + pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n", + prange->it_node.start, prange->it_node.last); + } + +out_free: + kvfree(migrate.src); +out: + return r; +} + +/** + * svm_migrate_vram_to_ram - migrate svm range from device to system + * @prange: range structure + * @mm: process mm, use current->mm if NULL + * + * Context: Process context, caller hold mm->mmap_sem and prange->lock and take + * svms srcu read lock + * + * Return: + * 0 - OK, otherwise error code + */ +int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm) +{ + struct amdgpu_device *adev; + struct vm_area_struct *vma; + unsigned long addr; + unsigned long start; + unsigned long end; + int r = 0; + + if (!prange->actual_loc) { + pr_debug("[0x%lx 0x%lx] already migrated to ram\n", + prange->it_node.start, prange->it_node.last); + return 0; + } + + adev = svm_range_get_adev_by_id(prange, prange->actual_loc); + if (!adev) { + pr_debug("failed to get device by id 0x%x\n", + prange->actual_loc); + return -ENODEV; + } + + pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + + start = prange->it_node.start << PAGE_SHIFT; + end = (prange->it_node.last + 1) << PAGE_SHIFT; + + for (addr = start; addr < end;) { + unsigned long next; + + vma = find_vma(mm, addr); + if (!vma || addr < vma->vm_start) + break; + + next = min(vma->vm_end, end); + r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next); + if (r) { + pr_debug("failed %d to migrate\n", r); + break; + } + addr = next; + } + + svm_range_vram_node_free(prange); + prange->actual_loc = 0; + + return r; }

/** * svm_migrate_to_ram - CPU page fault handler * @vmf: CPU vm fault vma, address * - * Context: vm fault handler, mm->mmap_sem is taken + * Context: vm fault handler, caller holds the mmap lock * * Return: * 0 - OK @@ -485,7 +709,53 @@ static void svm_migrate_page_free(struct page *page) */ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf) { - return VM_FAULT_SIGBUS; + unsigned long addr = vmf->address; + struct vm_area_struct *vma; + struct svm_range *prange; + struct list_head list; + struct kfd_process *p; + int r = VM_FAULT_SIGBUS; + int srcu_idx; + + vma = vmf->vma; + + p = kfd_lookup_process_by_mm(vma->vm_mm); + if (!p) { + pr_debug("failed find process at fault address 0x%lx\n", addr); + return VM_FAULT_SIGBUS; + } + + /* To prevent prange is removed */ + srcu_idx = srcu_read_lock(&p->svms.srcu); + + addr >>= PAGE_SHIFT; + pr_debug("CPU page fault svms 0x%p address 0x%lx\n", &p->svms, addr); + + r = svm_range_split_by_granularity(p, addr, &list); + if (r) { + pr_debug("failed %d to split range by granularity\n", r); + goto out_srcu; + } + + list_for_each_entry(prange, &list, update_list) { + mutex_lock(&prange->mutex); + r = svm_migrate_vram_to_ram(prange, vma->vm_mm); + mutex_unlock(&prange->mutex); + if (r) { + pr_debug("failed %d migrate [0x%lx 0x%lx] to ram\n", r, + prange->it_node.start, prange->it_node.last); + goto out_srcu; + } + } + +out_srcu: + srcu_read_unlock(&p->svms.srcu, srcu_idx); + kfd_unref_process(p); + + if (r) + return VM_FAULT_SIGBUS; + + return 0; }

static const struct dev_pagemap_ops svm_migrate_pgmap_ops = { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h index ffae5f989909..95fd7b21791f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h @@ -38,6 +38,9 @@ enum MIGRATION_COPY_DIR { };

int svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc); +int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm); +unsigned long +svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr);

#if defined(CONFIG_DEVICE_PRIVATE) int svm_migrate_init(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 0dbc403413a1..37f35f986930 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -819,6 +819,92 @@ svm_range_split_add_front(struct svm_range *prange, struct svm_range *new, return 0; }

+/** + * svm_range_split_by_granularity - collect ranges within granularity boundary + * + * @p: the process with svms list + * @addr: the vm fault address in pages, to search ranges + * @list: output, the range list + * + * Collects small ranges that make up one migration granule and splits the first + * and the last range at the granularity boundary + * + * Context: hold and release svms lock + * + * Return: + * 0 - OK, otherwise error code + */ +int svm_range_split_by_granularity(struct kfd_process *p, unsigned long addr, + struct list_head *list) +{ + struct svm_range *prange; + struct svm_range *tail; + struct svm_range *new; + unsigned long start; + unsigned long last; + unsigned long size; + int r = 0; + + svms_lock(&p->svms); + + prange = svm_range_from_addr(&p->svms, addr); + if (!prange) { + pr_debug("cannot find svm range at 0x%lx\n", addr); + svms_unlock(&p->svms); + return -EFAULT; + } + + /* Align splited range start and size to granularity size, then a single + * PTE will be used for whole range, this reduces the number of PTE + * updated and the L1 TLB space used for translation. + */ + size = 1ULL << prange->granularity; + start = ALIGN_DOWN(addr, size); + last = ALIGN(addr + 1, size) - 1; + INIT_LIST_HEAD(list); + + pr_debug("svms 0x%p split [0x%lx 0x%lx] at 0x%lx granularity 0x%lx\n", + prange->svms, start, last, addr, size); + + if (start > prange->it_node.start) { + r = svm_range_split(prange, prange->it_node.start, start - 1, + &new); + if (r) + goto out_unlock; + + svm_range_add_to_svms(new); + } else { + new = prange; + } + + while (size > new->npages) { + struct interval_tree_node *next; + + list_add(&new->update_list, list); + + next = interval_tree_iter_next(&new->it_node, start, last); + if (!next) + goto out_unlock; + + size -= new->npages; + new = container_of(next, struct svm_range, it_node); + } + + if (last < new->it_node.last) { + r = svm_range_split(new, new->it_node.start, last, &tail); + if (r) + goto out_unlock; + svm_range_add_to_svms(tail); + } + + list_add(&new->update_list, list); + +out_unlock: + svms_unlock(&p->svms); + + return r; +} + static uint64_t svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange) { @@ -1508,6 +1594,27 @@ static const struct mmu_interval_notifier_ops svm_range_mn_ops = { .invalidate = svm_range_cpu_invalidate_pagetables, };

+/** + * svm_range_from_addr - find svm range from fault address + * @svms: svm range list header + * @addr: address to search range interval tree, in pages + * + * Context: The caller must hold svms_lock + * + * Return: the svm_range found or NULL + */ +struct svm_range * +svm_range_from_addr(struct svm_range_list *svms, unsigned long addr) +{ + struct interval_tree_node *node; + + node = interval_tree_iter_first(&svms->objects, addr, addr); + if (!node) + return NULL; + + return container_of(node, struct svm_range, it_node); +} + void svm_range_list_fini(struct kfd_process *p) { pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms); @@ -1754,11 +1861,14 @@ svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange,

pr_debug("migrate from ram to vram\n"); r = svm_migrate_ram_to_vram(prange, best_loc); - - if (!r) - *migrated = true; + } else { + pr_debug("migrate from vram to ram\n"); + r = svm_migrate_vram_to_ram(prange, current->mm); }

+ if (!r) + *migrated = true; + return r; }

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index b81dfb32135b..c67e96f764fe 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -112,10 +112,14 @@ void svm_range_list_fini(struct kfd_process *p); int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start, uint64_t size, uint32_t nattrs, struct kfd_ioctl_svm_attribute *attrs); +struct svm_range *svm_range_from_addr(struct svm_range_list *svms, + unsigned long addr); struct amdgpu_device *svm_range_get_adev_by_id(struct svm_range *prange, uint32_t id); int svm_range_vram_node_new(struct amdgpu_device *adev, struct svm_range *prange, bool clear); void svm_range_vram_node_free(struct svm_range *prange); +int svm_range_split_by_granularity(struct kfd_process *p, unsigned long addr, + struct list_head *list);

#endif /* KFD_SVM_H_ */

From: Alex Sierra alex.sierra@amd.com

Page table restore implementation in SVM API. This is called from the fault handler at amdgpu_vm. To update page tables through the page fault retry IH.

Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 78 ++++++++++++++++++++++++++++ drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 2 + 2 files changed, 80 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index ea27c5ed4ef3..7346255f7c27 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -1629,6 +1629,84 @@ svm_range_from_addr(struct svm_range_list *svms, unsigned long addr) return container_of(node, struct svm_range, it_node); }

+int +svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, + uint64_t addr) +{ + int r = 0; + int srcu_idx; + struct mm_struct *mm = NULL; + struct svm_range *prange; + struct svm_range_list *svms; + struct kfd_process *p; + + p = kfd_lookup_process_by_pasid(pasid); + if (!p) { + pr_debug("kfd process not founded pasid 0x%x\n", pasid); + return -ESRCH; + } + svms = &p->svms; + srcu_idx = srcu_read_lock(&svms->srcu); + + pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr); + + svms_lock(svms); + prange = svm_range_from_addr(svms, addr); + svms_unlock(svms); + if (!prange) { + pr_debug("failed to find prange svms 0x%p address [0x%llx]\n", + svms, addr); + r = -EFAULT; + goto unlock_out; + } + + if (!atomic_read(&prange->invalid)) { + pr_debug("svms 0x%p [0x%lx %lx] already restored\n", + svms, prange->it_node.start, prange->it_node.last); + goto unlock_out; + } + + mm = get_task_mm(p->lead_thread); + if (!mm) { + pr_debug("svms 0x%p failed to get mm\n", svms); + r = -ESRCH; + goto unlock_out; + } + + mmap_read_lock(mm); + + /* + * If range is migrating, wait for migration is done. + */ + mutex_lock(&prange->mutex); + + r = svm_range_validate(mm, prange); + if (r) { + pr_debug("failed %d to validate svms 0x%p [0x%lx 0x%lx]\n", r, + svms, prange->it_node.start, prange->it_node.last); + + goto mmput_out; + } + + pr_debug("restoring svms 0x%p [0x%lx %lx] mapping\n", + svms, prange->it_node.start, prange->it_node.last); + + r = svm_range_map_to_gpus(prange, true); + if (r) + pr_debug("failed %d to map svms 0x%p [0x%lx 0x%lx] to gpu\n", r, + svms, prange->it_node.start, prange->it_node.last); + +mmput_out: + mutex_unlock(&prange->mutex); + mmap_read_unlock(mm); + mmput(mm); +unlock_out: + srcu_read_unlock(&svms->srcu, srcu_idx); + kfd_unref_process(p); + + return r; +} + void svm_range_list_fini(struct kfd_process *p) { pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index c67e96f764fe..e546f36ef709 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -121,5 +121,7 @@ int svm_range_vram_node_new(struct amdgpu_device *adev, void svm_range_vram_node_free(struct svm_range *prange); int svm_range_split_by_granularity(struct kfd_process *p, unsigned long addr, struct list_head *list); +int svm_range_restore_pages(struct amdgpu_device *adev, + unsigned int pasid, uint64_t addr);

#endif /* KFD_SVM_H_ */

From: Alex Sierra alex.sierra@amd.com

[why] As part of the SVM functionality, the eviction mechanism used for SVM_BOs is different. This mechanism uses one eviction fence per prange, instead of one fence per kfd_process.

[how] A svm_bo reference to amdgpu_amdkfd_fence to allow differentiate between SVM_BO or regular BO evictions. This also include modifications to set the reference at the fence creation call.

Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 4 +++- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c | 5 +++-- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 6 ++++-- 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index bc9f0e42e0a2..fb8be788ac1b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -75,6 +75,7 @@ struct amdgpu_amdkfd_fence { struct mm_struct *mm; spinlock_t lock; char timeline_name[TASK_COMM_LEN]; + struct svm_range_bo *svm_bo; };

struct amdgpu_kfd_dev { @@ -95,7 +96,8 @@ enum kgd_engine_type { };

struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, - struct mm_struct *mm); + struct mm_struct *mm, + struct svm_range_bo *svm_bo); bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm); struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f); int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c index 3107b9575929..9cc85efa4ed5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c @@ -60,7 +60,8 @@ static atomic_t fence_seq = ATOMIC_INIT(0); */

struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, - struct mm_struct *mm) + struct mm_struct *mm, + struct svm_range_bo *svm_bo) { struct amdgpu_amdkfd_fence *fence;

@@ -73,7 +74,7 @@ struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, fence->mm = mm; get_task_comm(fence->timeline_name, current); spin_lock_init(&fence->lock); - + fence->svm_bo = svm_bo; dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock, context, atomic_inc_return(&fence_seq));

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 99ad4e1d0896..8a43f3880022 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -928,7 +928,8 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,

info->eviction_fence = amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1), - current->mm); + current->mm, + NULL); if (!info->eviction_fence) { pr_err("Failed to create eviction fence\n"); ret = -ENOMEM; @@ -2150,7 +2151,8 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef) */ new_fence = amdgpu_amdkfd_fence_create( process_info->eviction_fence->base.context, - process_info->eviction_fence->mm); + process_info->eviction_fence->mm, + NULL); if (!new_fence) { pr_err("Failed to create eviction fence\n"); ret = -ENOMEM;

From: Alex Sierra alex.sierra@amd.com

svm_bo eviction mechanism is different from regular BOs. Every SVM_BO created contains one eviction fence and one worker item for eviction process. SVM_BOs can be attached to one or more pranges. For SVM_BO eviction mechanism, TTM will start to call enable_signal callback for every SVM_BO until VRAM space is available. Here, all the ttm_evict calls are synchronous, this guarantees that each eviction has completed and the fence has signaled before it returns.

Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 197 ++++++++++++++++++++------- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 13 +- 2 files changed, 160 insertions(+), 50 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 7346255f7c27..63b745a06740 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -34,6 +34,7 @@

#define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1

+static void svm_range_evict_svm_bo_worker(struct work_struct *work); /** * svm_range_unlink - unlink svm_range from lists and interval tree * @prange: svm range structure to be removed @@ -260,7 +261,15 @@ static void svm_range_bo_release(struct kref *kref) list_del_init(&prange->svm_bo_list); } spin_unlock(&svm_bo->list_lock); - + if (!dma_fence_is_signaled(&svm_bo->eviction_fence->base)) { + /* We're not in the eviction worker. + * Signal the fence and synchronize with any + * pending eviction work. + */ + dma_fence_signal(&svm_bo->eviction_fence->base); + cancel_work_sync(&svm_bo->eviction_work); + } + dma_fence_put(&svm_bo->eviction_fence->base); amdgpu_bo_unref(&svm_bo->bo); kfree(svm_bo); } @@ -273,6 +282,62 @@ static void svm_range_bo_unref(struct svm_range_bo *svm_bo) kref_put(&svm_bo->kref, svm_range_bo_release); }

+static bool svm_range_validate_svm_bo(struct svm_range *prange) +{ + spin_lock(&prange->svm_bo_lock); + if (!prange->svm_bo) { + spin_unlock(&prange->svm_bo_lock); + return false; + } + if (prange->mm_nodes) { + /* We still have a reference, all is well */ + spin_unlock(&prange->svm_bo_lock); + return true; + } + if (svm_bo_ref_unless_zero(prange->svm_bo)) { + if (READ_ONCE(prange->svm_bo->evicting)) { + struct dma_fence *f; + struct svm_range_bo *svm_bo; + /* The BO is getting evicted, + * we need to get a new one + */ + spin_unlock(&prange->svm_bo_lock); + svm_bo = prange->svm_bo; + f = dma_fence_get(&svm_bo->eviction_fence->base); + svm_range_bo_unref(prange->svm_bo); + /* wait for the fence to avoid long spin-loop + * at list_empty_careful + */ + dma_fence_wait(f, false); + dma_fence_put(f); + } else { + /* The BO was still around and we got + * a new reference to it + */ + spin_unlock(&prange->svm_bo_lock); + pr_debug("reuse old bo svms 0x%p [0x%lx 0x%lx]\n", + prange->svms, prange->it_node.start, + prange->it_node.last); + + prange->mm_nodes = prange->svm_bo->bo->tbo.mem.mm_node; + return true; + } + + } else { + spin_unlock(&prange->svm_bo_lock); + } + + /* We need a new svm_bo. Spin-loop to wait for concurrent + * svm_range_bo_release to finish removing this range from + * its range list. After this, it is safe to reuse the + * svm_bo pointer and svm_bo_list head. + */ + while (!list_empty_careful(&prange->svm_bo_list)) + ; + + return false; +} + static struct svm_range_bo *svm_range_bo_new(void) { struct svm_range_bo *svm_bo; @@ -292,71 +357,54 @@ int svm_range_vram_node_new(struct amdgpu_device *adev, struct svm_range *prange, bool clear) { - struct amdkfd_process_info *process_info; struct amdgpu_bo_param bp; struct svm_range_bo *svm_bo; struct amdgpu_bo *bo; struct kfd_process *p; + struct mm_struct *mm; int r;

- pr_debug("[0x%lx 0x%lx]\n", prange->it_node.start, - prange->it_node.last); - spin_lock(&prange->svm_bo_lock); - if (prange->svm_bo) { - if (prange->mm_nodes) { - /* We still have a reference, all is well */ - spin_unlock(&prange->svm_bo_lock); - return 0; - } - if (svm_bo_ref_unless_zero(prange->svm_bo)) { - /* The BO was still around and we got - * a new reference to it - */ - spin_unlock(&prange->svm_bo_lock); - pr_debug("reuse old bo [0x%lx 0x%lx]\n", - prange->it_node.start, prange->it_node.last); - - prange->mm_nodes = prange->svm_bo->bo->tbo.mem.mm_node; - return 0; - } - - spin_unlock(&prange->svm_bo_lock); - - /* We need a new svm_bo. Spin-loop to wait for concurrent - * svm_range_bo_release to finish removing this range from - * its range list. After this, it is safe to reuse the - * svm_bo pointer and svm_bo_list head. - */ - while (!list_empty_careful(&prange->svm_bo_list)) - ; + p = container_of(prange->svms, struct kfd_process, svms); + pr_debug("pasid: %x svms 0x%p [0x%lx 0x%lx]\n", p->pasid, prange->svms, + prange->it_node.start, prange->it_node.last);

- } else { - spin_unlock(&prange->svm_bo_lock); - } + if (svm_range_validate_svm_bo(prange)) + return 0;

svm_bo = svm_range_bo_new(); if (!svm_bo) { pr_debug("failed to alloc svm bo\n"); return -ENOMEM; } - + mm = get_task_mm(p->lead_thread); + if (!mm) { + pr_debug("failed to get mm\n"); + kfree(svm_bo); + return -ESRCH; + } + svm_bo->svms = prange->svms; + svm_bo->eviction_fence = + amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1), + mm, + svm_bo); + mmput(mm); + INIT_WORK(&svm_bo->eviction_work, svm_range_evict_svm_bo_worker); + svm_bo->evicting = 0; memset(&bp, 0, sizeof(bp)); bp.size = prange->npages * PAGE_SIZE; bp.byte_align = PAGE_SIZE; bp.domain = AMDGPU_GEM_DOMAIN_VRAM; bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS; bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0; + bp.flags |= AMDGPU_AMDKFD_CREATE_SVM_BO; bp.type = ttm_bo_type_device; bp.resv = NULL;

r = amdgpu_bo_create(adev, &bp, &bo); if (r) { pr_debug("failed %d to create bo\n", r); - kfree(svm_bo); - return r; + goto create_bo_failed; } - - p = container_of(prange->svms, struct kfd_process, svms); r = amdgpu_bo_reserve(bo, true); if (r) { pr_debug("failed %d to reserve bo\n", r); @@ -369,8 +417,7 @@ svm_range_vram_node_new(struct amdgpu_device *adev, struct svm_range *prange, amdgpu_bo_unreserve(bo); goto reserve_bo_failed; } - process_info = p->kgd_process_info; - amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true); + amdgpu_bo_fence(bo, &svm_bo->eviction_fence->base, true);

amdgpu_bo_unreserve(bo);

@@ -380,14 +427,16 @@ svm_range_vram_node_new(struct amdgpu_device *adev, struct svm_range *prange, prange->offset = 0;

spin_lock(&svm_bo->list_lock); - list_add(&prange->svm_bo_list, &svm_bo->range_list); + list_add_rcu(&prange->svm_bo_list, &svm_bo->range_list); spin_unlock(&svm_bo->list_lock);

return 0;

reserve_bo_failed: - kfree(svm_bo); amdgpu_bo_unref(&bo); +create_bo_failed: + dma_fence_put(&svm_bo->eviction_fence->base); + kfree(svm_bo); prange->mm_nodes = NULL;

return r; @@ -621,7 +670,7 @@ svm_range_split_nodes(struct svm_range *new, struct svm_range *old, new->mm_nodes = old->mm_nodes;

spin_lock(&new->svm_bo->list_lock); - list_add(&new->svm_bo_list, &new->svm_bo->range_list); + list_add_rcu(&new->svm_bo_list, &new->svm_bo->range_list); spin_unlock(&new->svm_bo->list_lock);

return 0; @@ -1353,7 +1402,7 @@ struct svm_range *svm_range_clone(struct svm_range *old) new->offset = old->offset; new->svm_bo = svm_range_bo_ref(old->svm_bo); spin_lock(&new->svm_bo->list_lock); - list_add(&new->svm_bo_list, &new->svm_bo->range_list); + list_add_rcu(&new->svm_bo_list, &new->svm_bo->range_list); spin_unlock(&new->svm_bo->list_lock); } new->flags = old->flags; @@ -1964,6 +2013,62 @@ svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange, return r; }

+int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence) +{ + if (!fence) + return -EINVAL; + + if (dma_fence_is_signaled(&fence->base)) + return 0; + + if (fence->svm_bo) { + WRITE_ONCE(fence->svm_bo->evicting, 1); + schedule_work(&fence->svm_bo->eviction_work); + } + + return 0; +} + +static void svm_range_evict_svm_bo_worker(struct work_struct *work) +{ + struct svm_range_bo *svm_bo; + struct svm_range *prange; + struct kfd_process *p; + struct mm_struct *mm; + int srcu_idx; + + svm_bo = container_of(work, struct svm_range_bo, eviction_work); + if (!svm_bo_ref_unless_zero(svm_bo)) + return; /* svm_bo was freed while eviction was pending */ + + /* svm_range_bo_release destroys this worker thread. So during + * the lifetime of this thread, kfd_process and mm will be valid. + */ + p = container_of(svm_bo->svms, struct kfd_process, svms); + mm = p->mm; + if (!mm) + return; + + mmap_read_lock(mm); + srcu_idx = srcu_read_lock(&svm_bo->svms->srcu); + list_for_each_entry_rcu(prange, &svm_bo->range_list, svm_bo_list) { + pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + mutex_lock(&prange->mutex); + svm_migrate_vram_to_ram(prange, svm_bo->eviction_fence->mm); + mutex_unlock(&prange->mutex); + } + srcu_read_unlock(&svm_bo->svms->srcu, srcu_idx); + mmap_read_unlock(mm); + + dma_fence_signal(&svm_bo->eviction_fence->base); + /* This is the last reference to svm_bo, after svm_range_vram_node_free + * has been called in svm_migrate_vram_to_ram + */ + WARN_ONCE(kref_read(&svm_bo->kref) != 1, "This was not the last reference\n"); + svm_range_bo_unref(svm_bo); +} + static int svm_range_set_attr(struct kfd_process *p, uint64_t start, uint64_t size, uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index e546f36ef709..143573621956 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -33,10 +33,14 @@ #include "kfd_priv.h"

struct svm_range_bo { - struct amdgpu_bo *bo; - struct kref kref; - struct list_head range_list; /* all svm ranges shared this bo */ - spinlock_t list_lock; + struct amdgpu_bo *bo; + struct kref kref; + struct list_head range_list; /* all svm ranges shared this bo */ + spinlock_t list_lock; + struct amdgpu_amdkfd_fence *eviction_fence; + struct work_struct eviction_work; + struct svm_range_list *svms; + uint32_t evicting; }; /** * struct svm_range - shared virtual memory range @@ -123,5 +127,6 @@ int svm_range_split_by_granularity(struct kfd_process *p, unsigned long addr, struct list_head *list); int svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, uint64_t addr); +int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence);

#endif /* KFD_SVM_H_ */

From: Alex Sierra alex.sierra@amd.com

Add to amdgpu_amdkfd_fence.enable_signal callback, support for svm_bo fence eviction.

Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c index 9cc85efa4ed5..98d6e08f22d8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c @@ -28,6 +28,7 @@ #include <linux/slab.h> #include <linux/sched/mm.h> #include "amdgpu_amdkfd.h" +#include "kfd_svm.h"

static const struct dma_fence_ops amdkfd_fence_ops; static atomic_t fence_seq = ATOMIC_INIT(0); @@ -123,9 +124,13 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f) if (dma_fence_is_signaled(f)) return true;

- if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f)) - return true; - + if (!fence->svm_bo) { + if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f)) + return true; + } else { + if (!svm_range_schedule_evict_svm_bo(fence)) + return true; + } return false; }

From: Philip Yang Philip.Yang@amd.com

With xnack on, add validate timestamp in order to handle GPU vm fault from multiple GPUs.

If GPU retry fault need migrate the range to the best restore location, use range validate timestamp to record system timestamp after range is restored to update GPU page table.

Because multiple pages of same range have multiple retry fault, define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING to the long time period that pending retry fault may still comes after page table update, to skip duplicate retry fault of same range.

If difference between system timestamp and range last validate timestamp is bigger than AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING, that means the retry fault is from another GPU, then continue to handle retry fault recover.

Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 27 +++++++++++++++++++++++---- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 2 ++ 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 8b57f5a471bd..65f20a72ddcb 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -34,6 +34,11 @@

#define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1

+/* Long enough to ensure no retry fault comes after svm range is restored and + * page table is updated. + */ +#define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING 2000 + static void svm_range_evict_svm_bo_worker(struct work_struct *work); /** * svm_range_unlink - unlink svm_range from lists and interval tree @@ -122,6 +127,7 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start, INIT_LIST_HEAD(&prange->remove_list); INIT_LIST_HEAD(&prange->svm_bo_list); atomic_set(&prange->invalid, 0); + prange->validate_timestamp = ktime_to_us(ktime_get()); mutex_init(&prange->mutex); spin_lock_init(&prange->svm_bo_lock); svm_range_set_default_attributes(&prange->preferred_loc, @@ -482,20 +488,28 @@ static int svm_range_validate_vram(struct svm_range *prange) static int svm_range_validate(struct mm_struct *mm, struct svm_range *prange) { + struct kfd_process *p; int r;

pr_debug("svms 0x%p [0x%lx 0x%lx] actual loc 0x%x\n", prange->svms, prange->it_node.start, prange->it_node.last, prange->actual_loc);

+ p = container_of(prange->svms, struct kfd_process, svms); + if (!prange->actual_loc) r = svm_range_validate_ram(mm, prange); else r = svm_range_validate_vram(prange);

- pr_debug("svms 0x%p [0x%lx 0x%lx] ret %d invalid %d\n", prange->svms, - prange->it_node.start, prange->it_node.last, - r, atomic_read(&prange->invalid)); + if (!r) { + if (p->xnack_enabled) + atomic_set(&prange->invalid, 0); + prange->validate_timestamp = ktime_to_us(ktime_get()); + } + + pr_debug("svms 0x%p [0x%lx 0x%lx] ret %d\n", prange->svms, + prange->it_node.start, prange->it_node.last, r);

return r; } @@ -1766,6 +1780,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct svm_range_list *svms; struct svm_range *prange; struct kfd_process *p; + uint64_t timestamp; int32_t best_loc; int srcu_idx; int r = 0; @@ -1790,7 +1805,11 @@ svm_range_restore_pages(struct amdgpu_device *adev, struct amdgpu_vm *vm, goto out_srcu_unlock; }

- if (!atomic_read(&prange->invalid)) { + mutex_lock(&prange->mutex); + timestamp = ktime_to_us(ktime_get()) - prange->validate_timestamp; + mutex_unlock(&prange->mutex); + /* skip duplicate vm fault on different pages of same range */ + if (timestamp < AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING) { pr_debug("svms 0x%p [0x%lx %lx] already restored\n", svms, prange->it_node.start, prange->it_node.last); goto out_srcu_unlock; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index 0685eb04b87c..466ec5537bbb 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -66,6 +66,7 @@ struct svm_range_bo { * @actual_loc: the actual location, 0 for CPU, or GPU id * @granularity:migration granularity, log2 num pages * @invalid: not 0 means cpu page table is invalidated + * @validate_timestamp: system timestamp when range is validated * @bitmap_access: index bitmap of GPUs which can access the range * @bitmap_aip: index bitmap of GPUs which can access the range in place * @@ -95,6 +96,7 @@ struct svm_range { uint32_t actual_loc; uint8_t granularity; atomic_t invalid; + uint64_t validate_timestamp; DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE); DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE); };