From: Philip Yang Philip.Yang@amd.com
If svm range perfetch location is not zero, use TTM to alloc amdgpu_bo vram nodes to validate svm range, then map vram nodes to GPUs.
Use offset to sub allocate from the same amdgpu_bo to handle overlap vram range while adding new range or unmapping range.
svm_bo has ref count to trace the shared ranges. If all ranges of shared amdgpu_bo are migrated to ram, ref count becomes 0, then amdgpu_bo is released, all ranges svm_bo is set to NULL.
To migrate range from ram back to vram, allocate the same amdgpu_bo with previous offset if the range has svm_bo.
Signed-off-by: Philip Yang Philip.Yang@amd.com Signed-off-by: Alex Sierra alex.sierra@amd.com Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 342 ++++++++++++++++++++++++--- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 20 ++ 2 files changed, 335 insertions(+), 27 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index e3ba6e7262a7..7d91dc49a5a9 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -35,7 +35,9 @@ * svm_range_unlink - unlink svm_range from lists and interval tree * @prange: svm range structure to be removed * - * Remove the svm range from svms interval tree and link list + * Remove the svm_range from the svms and svm_bo SRCU lists and the svms + * interval tree. After this call, synchronize_srcu is needed before the + * range can be freed safely. * * Context: The caller must hold svms_lock */ @@ -44,6 +46,12 @@ static void svm_range_unlink(struct svm_range *prange) pr_debug("prange 0x%p [0x%lx 0x%lx]\n", prange, prange->it_node.start, prange->it_node.last);
+ if (prange->svm_bo) { + spin_lock(&prange->svm_bo->list_lock); + list_del(&prange->svm_bo_list); + spin_unlock(&prange->svm_bo->list_lock); + } + list_del_rcu(&prange->list); interval_tree_remove(&prange->it_node, &prange->svms->objects); } @@ -70,6 +78,12 @@ static void svm_range_remove(struct svm_range *prange) pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->it_node.start, prange->it_node.last);
+ if (prange->mm_nodes) { + pr_debug("vram prange svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + svm_range_vram_node_free(prange); + } + kvfree(prange->pages_addr); kfree(prange); } @@ -102,7 +116,9 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start, INIT_LIST_HEAD(&prange->list); INIT_LIST_HEAD(&prange->update_list); INIT_LIST_HEAD(&prange->remove_list); + INIT_LIST_HEAD(&prange->svm_bo_list); atomic_set(&prange->invalid, 0); + spin_lock_init(&prange->svm_bo_lock); svm_range_set_default_attributes(&prange->preferred_loc, &prange->prefetch_loc, &prange->granularity, &prange->flags); @@ -139,6 +155,16 @@ svm_get_supported_dev_by_id(struct kfd_process *p, uint32_t gpu_id, return dev; }
+struct amdgpu_device * +svm_range_get_adev_by_id(struct svm_range *prange, uint32_t gpu_id) +{ + struct kfd_process *p = + container_of(prange->svms, struct kfd_process, svms); + struct kfd_dev *dev = svm_get_supported_dev_by_id(p, gpu_id, NULL); + + return dev ? (struct amdgpu_device *)dev->kgd : NULL; +} + /** * svm_range_validate_ram - get system memory pages of svm range * @@ -186,14 +212,226 @@ svm_range_validate_ram(struct mm_struct *mm, struct svm_range *prange) return 0; }
+static bool svm_bo_ref_unless_zero(struct svm_range_bo *svm_bo) +{ + if (!svm_bo || !kref_get_unless_zero(&svm_bo->kref)) + return false; + + return true; +} + +static struct svm_range_bo *svm_range_bo_ref(struct svm_range_bo *svm_bo) +{ + if (svm_bo) + kref_get(&svm_bo->kref); + + return svm_bo; +} + +static void svm_range_bo_release(struct kref *kref) +{ + struct svm_range_bo *svm_bo; + + svm_bo = container_of(kref, struct svm_range_bo, kref); + /* This cleanup loop does not need to be SRCU safe because there + * should be no SRCU readers while the ref count is 0. Any SRCU + * reader that has a chance of reducing the ref count must take + * an extra reference before srcu_read_lock and release it after + * srcu_read_unlock. + */ + spin_lock(&svm_bo->list_lock); + while (!list_empty(&svm_bo->range_list)) { + struct svm_range *prange = + list_first_entry(&svm_bo->range_list, + struct svm_range, svm_bo_list); + pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + spin_lock(&prange->svm_bo_lock); + prange->svm_bo = NULL; + spin_unlock(&prange->svm_bo_lock); + + /* list_del_init tells a concurrent svm_range_vram_node_new when + * it's safe to reuse the svm_bo pointer and svm_bo_list head. + */ + list_del_init(&prange->svm_bo_list); + } + spin_unlock(&svm_bo->list_lock); + + amdgpu_bo_unref(&svm_bo->bo); + kfree(svm_bo); +} + +static void svm_range_bo_unref(struct svm_range_bo *svm_bo) +{ + if (!svm_bo) + return; + + kref_put(&svm_bo->kref, svm_range_bo_release); +} + +static struct svm_range_bo *svm_range_bo_new(void) +{ + struct svm_range_bo *svm_bo; + + svm_bo = kzalloc(sizeof(*svm_bo), GFP_KERNEL); + if (!svm_bo) + return NULL; + + kref_init(&svm_bo->kref); + INIT_LIST_HEAD(&svm_bo->range_list); + spin_lock_init(&svm_bo->list_lock); + + return svm_bo; +} + +int +svm_range_vram_node_new(struct amdgpu_device *adev, struct svm_range *prange, + bool clear) +{ + struct amdkfd_process_info *process_info; + struct amdgpu_bo_param bp; + struct svm_range_bo *svm_bo; + struct amdgpu_bo *bo; + struct kfd_process *p; + int r; + + pr_debug("[0x%lx 0x%lx]\n", prange->it_node.start, + prange->it_node.last); + spin_lock(&prange->svm_bo_lock); + if (prange->svm_bo) { + if (prange->mm_nodes) { + /* We still have a reference, all is well */ + spin_unlock(&prange->svm_bo_lock); + return 0; + } + if (svm_bo_ref_unless_zero(prange->svm_bo)) { + /* The BO was still around and we got + * a new reference to it + */ + spin_unlock(&prange->svm_bo_lock); + pr_debug("reuse old bo [0x%lx 0x%lx]\n", + prange->it_node.start, prange->it_node.last); + + prange->mm_nodes = prange->svm_bo->bo->tbo.mem.mm_node; + return 0; + } + + spin_unlock(&prange->svm_bo_lock); + + /* We need a new svm_bo. Spin-loop to wait for concurrent + * svm_range_bo_release to finish removing this range from + * its range list. After this, it is safe to reuse the + * svm_bo pointer and svm_bo_list head. + */ + while (!list_empty_careful(&prange->svm_bo_list)) + ; + + } else { + spin_unlock(&prange->svm_bo_lock); + } + + svm_bo = svm_range_bo_new(); + if (!svm_bo) { + pr_debug("failed to alloc svm bo\n"); + return -ENOMEM; + } + + memset(&bp, 0, sizeof(bp)); + bp.size = prange->npages * PAGE_SIZE; + bp.byte_align = PAGE_SIZE; + bp.domain = AMDGPU_GEM_DOMAIN_VRAM; + bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS; + bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0; + bp.type = ttm_bo_type_device; + bp.resv = NULL; + + r = amdgpu_bo_create(adev, &bp, &bo); + if (r) { + pr_debug("failed %d to create bo\n", r); + kfree(svm_bo); + return r; + } + + p = container_of(prange->svms, struct kfd_process, svms); + r = amdgpu_bo_reserve(bo, true); + if (r) { + pr_debug("failed %d to reserve bo\n", r); + goto reserve_bo_failed; + } + + r = dma_resv_reserve_shared(bo->tbo.base.resv, 1); + if (r) { + pr_debug("failed %d to reserve bo\n", r); + amdgpu_bo_unreserve(bo); + goto reserve_bo_failed; + } + process_info = p->kgd_process_info; + amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true); + + amdgpu_bo_unreserve(bo); + + svm_bo->bo = bo; + prange->svm_bo = svm_bo; + prange->mm_nodes = bo->tbo.mem.mm_node; + prange->offset = 0; + + spin_lock(&svm_bo->list_lock); + list_add(&prange->svm_bo_list, &svm_bo->range_list); + spin_unlock(&svm_bo->list_lock); + + return 0; + +reserve_bo_failed: + kfree(svm_bo); + amdgpu_bo_unref(&bo); + prange->mm_nodes = NULL; + + return r; +} + +void svm_range_vram_node_free(struct svm_range *prange) +{ + pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, + prange->it_node.start, prange->it_node.last); + + svm_range_bo_unref(prange->svm_bo); + prange->mm_nodes = NULL; +} + +static int svm_range_validate_vram(struct svm_range *prange) +{ + struct amdgpu_device *adev; + int r; + + pr_debug("svms 0x%p [0x%lx 0x%lx] actual_loc 0x%x\n", prange->svms, + prange->it_node.start, prange->it_node.last, + prange->actual_loc); + + adev = svm_range_get_adev_by_id(prange, prange->actual_loc); + if (!adev) { + pr_debug("failed to get device by id 0x%x\n", + prange->actual_loc); + return -EINVAL; + } + + r = svm_range_vram_node_new(adev, prange, true); + if (r) + pr_debug("failed %d to alloc vram\n", r); + + return r; +} + static int svm_range_validate(struct mm_struct *mm, struct svm_range *prange) { - int r = 0; + int r;
pr_debug("actual loc 0x%x\n", prange->actual_loc);
- r = svm_range_validate_ram(mm, prange); + if (!prange->actual_loc) + r = svm_range_validate_ram(mm, prange); + else + r = svm_range_validate_vram(prange);
pr_debug("svms 0x%p [0x%lx 0x%lx] ret %d invalid %d\n", prange->svms, prange->it_node.start, prange->it_node.last, @@ -349,6 +587,35 @@ svm_range_split_pages(struct svm_range *new, struct svm_range *old, return 0; }
+static int +svm_range_split_nodes(struct svm_range *new, struct svm_range *old, + uint64_t start, uint64_t last) +{ + pr_debug("svms 0x%p new start 0x%lx start 0x%llx last 0x%llx\n", + new->svms, new->it_node.start, start, last); + + old->npages = last - start + 1; + + if (new->it_node.start == old->it_node.start) { + new->offset = old->offset; + old->offset += new->npages; + } else { + new->offset = old->offset + old->npages; + } + + old->it_node.start = start; + old->it_node.last = last; + + new->svm_bo = svm_range_bo_ref(old->svm_bo); + new->mm_nodes = old->mm_nodes; + + spin_lock(&new->svm_bo->list_lock); + list_add(&new->svm_bo_list, &new->svm_bo->range_list); + spin_unlock(&new->svm_bo->list_lock); + + return 0; +} + /** * svm_range_split_adjust - split range and adjust * @@ -382,6 +649,8 @@ svm_range_split_adjust(struct svm_range *new, struct svm_range *old,
if (old->pages_addr) r = svm_range_split_pages(new, old, start, last); + else if (old->actual_loc && old->mm_nodes) + r = svm_range_split_nodes(new, old, start, last); else WARN_ONCE(1, "split adjust invalid pages_addr and nodes\n"); if (r) @@ -438,17 +707,14 @@ svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last, return -EINVAL;
svms = prange->svms; - if (old_start == start) { + if (old_start == start) *new = svm_range_new(svms, last + 1, old_last); - if (!*new) - return -ENOMEM; - r = svm_range_split_adjust(*new, prange, start, last); - } else { + else *new = svm_range_new(svms, old_start, start - 1); - if (!*new) - return -ENOMEM; - r = svm_range_split_adjust(*new, prange, start, last); - } + if (!*new) + return -ENOMEM; + + r = svm_range_split_adjust(*new, prange, start, last);
return r; } @@ -550,7 +816,8 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange) uint64_t pte_flags;
pte_flags = AMDGPU_PTE_VALID; - pte_flags |= AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED; + if (!prange->mm_nodes) + pte_flags |= AMDGPU_PTE_SYSTEM | AMDGPU_PTE_SNOOPED;
mapping_flags = AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE;
@@ -570,7 +837,9 @@ svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange) /* Apply ASIC specific mapping flags */ amdgpu_gmc_get_vm_pte(adev, &prange->mapping, &pte_flags);
- pr_debug("PTE flags 0x%llx\n", pte_flags); + pr_debug("svms 0x%p [0x%lx 0x%lx] vram %d system %d PTE flags 0x%llx\n", + prange->svms, prange->it_node.start, prange->it_node.last, + prange->mm_nodes ? 1:0, prange->pages_addr ? 1:0, pte_flags);
return pte_flags; } @@ -656,7 +925,9 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct svm_range *prange, bool reserve_vm, struct dma_fence **fence) { - struct amdgpu_bo *root; + struct ttm_validate_buffer tv[2]; + struct ww_acquire_ctx ticket; + struct list_head list; dma_addr_t *pages_addr; uint64_t pte_flags; int r = 0; @@ -665,13 +936,25 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, prange->it_node.start, prange->it_node.last);
if (reserve_vm) { - root = amdgpu_bo_ref(vm->root.base.bo); - r = amdgpu_bo_reserve(root, true); + INIT_LIST_HEAD(&list); + + tv[0].bo = &vm->root.base.bo->tbo; + tv[0].num_shared = 4; + list_add(&tv[0].head, &list); + if (prange->svm_bo && prange->mm_nodes) { + tv[1].bo = &prange->svm_bo->bo->tbo; + tv[1].num_shared = 1; + list_add(&tv[1].head, &list); + } + r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL); if (r) { - pr_debug("failed %d to reserve root bo\n", r); - amdgpu_bo_unref(&root); + pr_debug("failed %d to reserve bo\n", r); goto out; } + if (prange->svm_bo && prange->mm_nodes && + prange->svm_bo->bo->tbo.evicted) + goto unreserve_out; + r = amdgpu_vm_validate_pt_bos(adev, vm, svm_range_bo_validate, NULL); if (r) { @@ -682,7 +965,7 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
prange->mapping.start = prange->it_node.start; prange->mapping.last = prange->it_node.last; - prange->mapping.offset = 0; + prange->mapping.offset = prange->offset; pte_flags = svm_range_get_pte_flags(adev, prange); prange->mapping.flags = pte_flags; pages_addr = prange->pages_addr; @@ -690,7 +973,8 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, r = amdgpu_vm_bo_update_mapping(adev, adev, vm, false, false, NULL, prange->mapping.start, prange->mapping.last, pte_flags, - prange->mapping.offset, NULL, + prange->mapping.offset, + prange->mm_nodes, pages_addr, &vm->last_update); if (r) { pr_debug("failed %d to map to gpu 0x%lx\n", r, @@ -710,11 +994,8 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, *fence = dma_fence_get(vm->last_update);
unreserve_out: - if (reserve_vm) { - amdgpu_bo_unreserve(root); - amdgpu_bo_unref(&root); - } - + if (reserve_vm) + ttm_eu_backoff_reservation(&ticket, &list); out: return r; } @@ -929,7 +1210,14 @@ struct svm_range *svm_range_clone(struct svm_range *old) memcpy(new->pages_addr, old->pages_addr, old->npages * sizeof(*old->pages_addr)); } - + if (old->svm_bo) { + new->mm_nodes = old->mm_nodes; + new->offset = old->offset; + new->svm_bo = svm_range_bo_ref(old->svm_bo); + spin_lock(&new->svm_bo->list_lock); + list_add(&new->svm_bo_list, &new->svm_bo->range_list); + spin_unlock(&new->svm_bo->list_lock); + } new->flags = old->flags; new->preferred_loc = old->preferred_loc; new->prefetch_loc = old->prefetch_loc; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index 4c7daf8e0b6f..b1d2db02043b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -32,6 +32,12 @@ #include "amdgpu.h" #include "kfd_priv.h"
+struct svm_range_bo { + struct amdgpu_bo *bo; + struct kref kref; + struct list_head range_list; /* all svm ranges shared this bo */ + spinlock_t list_lock; +}; /** * struct svm_range - shared virtual memory range * @@ -45,6 +51,10 @@ * @mapping: bo_va mapping structure to create and update GPU page table * @npages: number of pages * @pages_addr: list of system memory physical page address + * @mm_nodes: vram nodes allocated + * @offset: range start offset within mm_nodes + * @svm_bo: struct to manage splited amdgpu_bo + * @svm_bo_list:link list node, to scan all ranges which share same svm_bo * @flags: flags defined as KFD_IOCTL_SVM_FLAG_* * @perferred_loc: perferred location, 0 for CPU, or GPU id * @perfetch_loc: last prefetch location, 0 for CPU, or GPU id @@ -68,6 +78,11 @@ struct svm_range { struct amdgpu_bo_va_mapping mapping; uint64_t npages; dma_addr_t *pages_addr; + struct drm_mm_node *mm_nodes; + uint64_t offset; + struct svm_range_bo *svm_bo; + struct list_head svm_bo_list; + spinlock_t svm_bo_lock; uint32_t flags; uint32_t preferred_loc; uint32_t prefetch_loc; @@ -95,5 +110,10 @@ void svm_range_list_fini(struct kfd_process *p); int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start, uint64_t size, uint32_t nattrs, struct kfd_ioctl_svm_attribute *attrs); +struct amdgpu_device *svm_range_get_adev_by_id(struct svm_range *prange, + uint32_t id); +int svm_range_vram_node_new(struct amdgpu_device *adev, + struct svm_range *prange, bool clear); +void svm_range_vram_node_free(struct svm_range *prange);
#endif /* KFD_SVM_H_ */