Don't dma_unmap in unmap_from_gpu. The dma_addr arrays are protected by the migrate_mutex, which we cannot hold when unmapping in MMU notifiers.
Instead dma_unmap and free dma_addr arrays whenever the pages_array is invalidated: when migrating to VRAM and when re-validating RAM.
Freeing dma_addr arrays in svm_migrate_vma_to_vram fixes a bug where the stale system memory pages were mapped instead of VRAM after a migration.
When freeing dma_addr arrays, ignore the access bitmasks. Those masks may have changed since the dma_addr arrays were allocated and mapped.
Change-Id: I01a6121c4c9908c1da4f303e87dcafd509fabc86 Signed-off-by: Felix Kuehling Felix.Kuehling@amd.com --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 1 + drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 16 +++------------- drivers/gpu/drm/amd/amdkfd/kfd_svm.h | 1 + 3 files changed, 5 insertions(+), 13 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index cf71b98303c2..da2ff655812e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -444,6 +444,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange, prange->pages_addr = NULL;
svm_range_dma_unmap(adev->dev, scratch, 0, npages); + svm_range_free_dma_mappings(prange);
out_free: kvfree(buf); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 6c46d43bf613..3eea8f87724d 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -173,9 +173,8 @@ void svm_range_dma_unmap(struct device *dev, dma_addr_t *dma_addr, } }
-static void svm_range_free_dma_mappings(struct svm_range *prange) +void svm_range_free_dma_mappings(struct svm_range *prange) { - DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE); struct kfd_dev *kfd_dev; dma_addr_t *dma_addr; struct device *dev; @@ -184,13 +183,8 @@ static void svm_range_free_dma_mappings(struct svm_range *prange) int r;
p = container_of(prange->svms, struct kfd_process, svms); - if (p->xnack_enabled) - bitmap_copy(bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE); - else - bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip, - MAX_GPU_INSTANCE);
- for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) { + for (gpuidx = 0; gpuidx < MAX_GPU_INSTANCE; gpuidx++) { dma_addr = prange->dma_addr[gpuidx]; if (!dma_addr) continue; @@ -292,6 +286,7 @@ svm_range_validate_ram(struct mm_struct *mm, struct svm_range *prange) }
kvfree(prange->pages_addr); + svm_range_free_dma_mappings(prange); prange->pages_addr = prange->hmm_range->hmm_pfns; prange->hmm_range->hmm_pfns = NULL;
@@ -1192,11 +1187,6 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start, if (r) break; } - - svm_range_dma_unmap(adev->dev, prange->dma_addr[gpuidx], - start - prange->start, - last - start + 1); - amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev, p->pasid); } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index fea9c63b5f95..b2ab920ab884 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -175,5 +175,6 @@ void svm_range_add_list_work(struct svm_range_list *svms, void schedule_deferred_list_work(struct svm_range_list *svms); void svm_range_dma_unmap(struct device *dev, dma_addr_t *dma_addr, unsigned long offset, unsigned long npages); +void svm_range_free_dma_mappings(struct svm_range *prange);
#endif /* KFD_SVM_H_ */