From: Christian König christian.koenig@amd.com
Keep VM page tables behind normal BOs on the LRU.
Signed-off-by: Christian König christian.koenig@amd.com Acked-by: Alex Deucher alexander.deucher@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 53 +++++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index c4a21c6..1b5e7db 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -408,6 +408,10 @@ struct amdgpu_mman { struct amdgpu_ring *buffer_funcs_ring; /* Scheduler entity for buffer moves */ struct amd_sched_entity entity; + + /* custom LRU management */ + struct list_head *lru[TTM_NUM_MEM_TYPES]; + struct list_head *swap_lru; };
int amdgpu_copy_buffer(struct amdgpu_ring *ring, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index fefaa9b..f3d118b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -910,6 +910,49 @@ uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm, return flags; }
+static void amdgpu_ttm_lru_removal(struct ttm_buffer_object *tbo) +{ + struct amdgpu_device *adev = amdgpu_get_adev(tbo->bdev); + + if (&tbo->lru == adev->mman.lru[tbo->mem.mem_type]) + adev->mman.lru[tbo->mem.mem_type] = tbo->lru.prev; + + if (&tbo->swap == adev->mman.swap_lru) + adev->mman.swap_lru = tbo->swap.prev; +} + +static struct list_head *amdgpu_ttm_lru_tail(struct ttm_buffer_object *tbo) +{ + struct amdgpu_bo *bo = container_of(tbo, struct amdgpu_bo, tbo); + struct amdgpu_device *adev = amdgpu_get_adev(tbo->bdev); + struct list_head *res; + + /* only VM BOs have a parent set */ + if (bo->parent) + return ttm_bo_default_lru_tail(tbo); + + res = adev->mman.lru[tbo->mem.mem_type]; + adev->mman.lru[tbo->mem.mem_type] = &tbo->lru; + + return res; +} + +static struct list_head *amdgpu_ttm_swap_lru_tail(struct ttm_buffer_object *tbo) +{ + struct amdgpu_bo *bo = container_of(tbo, struct amdgpu_bo, tbo); + struct amdgpu_device *adev = amdgpu_get_adev(tbo->bdev); + struct list_head *res; + + /* only VM BOs have a parent set */ + if (bo->parent) + return ttm_bo_default_swap_lru_tail(tbo); + + res = adev->mman.swap_lru; + adev->mman.swap_lru = &tbo->swap; + + return res; +} + static struct ttm_bo_driver amdgpu_bo_driver = { .ttm_tt_create = &amdgpu_ttm_tt_create, .ttm_tt_populate = &amdgpu_ttm_tt_populate, @@ -923,12 +966,14 @@ static struct ttm_bo_driver amdgpu_bo_driver = { .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify, .io_mem_reserve = &amdgpu_ttm_io_mem_reserve, .io_mem_free = &amdgpu_ttm_io_mem_free, - .lru_tail = &ttm_bo_default_lru_tail, - .swap_lru_tail = &ttm_bo_default_swap_lru_tail, + .lru_removal = &amdgpu_ttm_lru_removal, + .lru_tail = &amdgpu_ttm_lru_tail, + .swap_lru_tail = &amdgpu_ttm_swap_lru_tail, };
int amdgpu_ttm_init(struct amdgpu_device *adev) { + unsigned i; int r;
r = amdgpu_ttm_global_init(adev); @@ -946,6 +991,10 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) DRM_ERROR("failed initializing buffer object driver(%d).\n", r); return r; } + for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) + adev->mman.lru[i] = &adev->mman.bdev.man[i].lru; + adev->mman.swap_lru = &adev->mman.bdev.glob->swap_lru; + adev->mman.initialized = true; r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM, adev->mc.real_vram_size >> PAGE_SHIFT);