general ttm lru cannot statisfy amdgpu per-vm-bo requirement, we have to adapt it in amdgpu driver at least.
Change-Id: I92b2286ef507c2e055ad9101cf31279d5f8db475 Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/ttm/ttm_bo.c | 54 ++++++++++++++++++++++++++++++----------- include/drm/ttm/ttm_bo_driver.h | 49 +++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 14 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 15506682a0be..98da2cf63c9b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -164,6 +164,8 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
reservation_object_assert_held(bo->resv);
+ if (bdev->driver->add_to_lru) + return bdev->driver->add_to_lru(bo); if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { BUG_ON(!list_empty(&bo->lru));
@@ -188,6 +190,8 @@ static void ttm_bo_ref_bug(struct kref *list_kref)
void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) { + struct ttm_bo_device *bdev = bo->bdev; + if (!list_empty(&bo->swap)) { list_del_init(&bo->swap); kref_put(&bo->list_kref, ttm_bo_ref_bug); @@ -201,6 +205,8 @@ void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) * TODO: Add a driver hook to delete from * driver-specific LRU's here. */ + if (bdev->driver->del_from_lru) + return bdev->driver->del_from_lru(bo); }
void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) @@ -215,10 +221,14 @@ EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { + struct ttm_bo_device *bdev = bo->bdev; + reservation_object_assert_held(bo->resv);
ttm_bo_del_from_lru(bo); ttm_bo_add_to_lru(bo); + if (bdev->driver->move_to_lru_tail) + return bdev->driver->move_to_lru_tail(bo); } EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
@@ -685,8 +695,8 @@ EXPORT_SYMBOL(ttm_bo_eviction_valuable); * * b. Otherwise, trylock it. */ -static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, - struct ttm_operation_ctx *ctx, bool *locked) +bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, bool *locked) { bool ret = false;
@@ -703,6 +713,7 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
return ret; } +EXPORT_SYMBOL(ttm_bo_evict_swapout_allowable);
static struct ttm_buffer_object * ttm_mem_get_evictable_bo(struct ttm_bo_device *bdev, @@ -736,6 +747,9 @@ ttm_mem_get_evictable_bo(struct ttm_bo_device *bdev, bo = NULL; }
+ if (!bo && bdev->driver->get_evictable_bo) + bo= bdev->driver->get_evictable_bo(bdev, mem_type, place, + ctx, locked); return bo; }
@@ -1311,6 +1325,21 @@ int ttm_bo_create(struct ttm_bo_device *bdev, } EXPORT_SYMBOL(ttm_bo_create);
+bool ttm_lru_empty(struct ttm_bo_device *bdev, unsigned mem_type) +{ + struct ttm_mem_type_manager *man = &bdev->man[mem_type]; + int i; + + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + if (!list_empty(&man->lru[i])) + return false; + } + if (bdev->driver->lru_empty) + return bdev->driver->lru_empty(bdev, mem_type); + + return true; +} + static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, unsigned mem_type) { @@ -1323,21 +1352,18 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, struct ttm_bo_global *glob = bdev->glob; struct dma_fence *fence; int ret; - unsigned i;
/* * Can't use standard list traversal since we're unlocking. */
spin_lock(&glob->lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - while (!list_empty(&man->lru[i])) { - spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); - if (ret) - return ret; - spin_lock(&glob->lru_lock); - } + while (!ttm_lru_empty(bdev, mem_type)) { + spin_unlock(&glob->lru_lock); + ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); + if (ret) + return ret; + spin_lock(&glob->lru_lock); } spin_unlock(&glob->lru_lock);
@@ -1533,9 +1559,9 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) pr_debug("Delayed destroy list was clean\n");
spin_lock(&glob->lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) - if (list_empty(&bdev->man[0].lru[0])) - pr_debug("Swap list %d was clean\n", i); + for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) + if (ttm_lru_empty(bdev, i)) + pr_debug("lru list %d was clean\n", i); spin_unlock(&glob->lru_lock);
drm_vma_offset_manager_destroy(&bdev->vma_manager); diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 3234cc322e70..29339b0a2fd6 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -284,6 +284,52 @@ struct ttm_bo_driver { */ bool (*eviction_valuable)(struct ttm_buffer_object *bo, const struct ttm_place *place); + + /** + * struct ttm_bo_driver member get_evictable_bo + * + * @bdev: the buffer object device. + * @mem_type: memory type + * @place: placement we need room for + * @ctx: context for this evict with parameters + * @locked: return if the evictable bo is already locked. + * + * return an evictable bo for evicting. + */ + struct ttm_buffer_object *(*get_evictable_bo)(struct ttm_bo_device *bdev, + uint32_t mem_type, + const struct ttm_place *place, + struct ttm_operation_ctx *ctx, + bool *locked); + + /** + * struct ttm_bo_driver member add_to_lru + * + * @bo: the buffer object to be add + * + * add bo to driver specific lru + */ + void (*add_to_lru)(struct ttm_buffer_object *bo); + + /** + * struct ttm_bo_driver member del_from_lru + * + * @bo: the buffer object to be add + * + * delete bo from driver specific lru + */ + void (*del_from_lru)(struct ttm_buffer_object *bo); + + /** + * struct ttm_bo_driver member move_to_lru_tail + * + * @bo: the buffer object to be add + * + * move to driver specific lru tail + */ + void (*move_to_lru_tail)(struct ttm_buffer_object *bo); + + bool (*lru_empty)(struct ttm_bo_device *bdev, unsigned mem_type); /** * struct ttm_bo_driver member evict_flags: * @@ -760,6 +806,9 @@ int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem); void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem); +bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, + bool *locked); /** * ttm_bo_move_ttm *