Since per-process-bo feature is introduced, old lru isn't working for it. old lru order is depending on BO list order, which will be updated by bo list after every command submission. But for per-process-bo, which aren't in bo list, so it have no chance to refresh its order in lru. Which also will resulit in unstable performance for application. per-process-bo means they will be used by every this process, and validated automatically. Their order should be fixed.
Above reason, we introduce a per-process-lru instead of old lru. like struct definition: +struct ttm_process { + struct list_head process_list; + struct rb_root fixed_lru[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; + struct list_head dynamic_lru[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; + struct reservation_object *resv; + atomic64_t bo_index; +}; process_list is a list node to add bdev->proces_list. fixed_lru is to store per-process-bo. dyanmic_lru is to store non per-process-bo. resvation is this process pd root bo resvation. bo_index is to counter bo index, every bo beloning to this process will get an index.
Last, the patch set is working in progress, so I don't reorganize and clean up them yet. Sending them to community is to see if any other concern for this new machanism.
Tested result: the solution solves my issue F1 game performance isn't stable, and not introduce extra cpu overhead, the Talos game can prove it.
TODO: 1. patch organization. 2. some corner case need to handle(fini, bo destroy, force_clean) 3. clean up. 4. more palces can be improved if this machanism is accepted, like per-vm-lru don't need move at all if not destroy, maybe add a flag for evictalbe. 5. vm->evicted list in amdgpu can also be replaced by RB tree, which can make sure validation order.
Thanks, David Zhou
Chunming Zhou (11): drm/ttm: add ttm process struct drm/amdgpu: use ttm process in amdgpu vm drm/amdgpu: add kernel process drm/amdgpu: pass process to tbo drm/ttm: add per process lru drm/amdgpu: pass ttm process to buffer object drm/ttm: use RB tree instead of link list drm/ttm: add bo index drm/amdgpu: counter for every bo creation drm/ttm: some good fixes for per-vm-lru drm/ttm: bulk move per vm bo
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 5 +- drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 24 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 169 ++++++++++++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 6 + drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 2 + drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 3 + drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 2 + drivers/gpu/drm/ttm/ttm_bo.c | 134 ++++++++++++++++++++--- include/drm/ttm/ttm_bo_api.h | 4 +- include/drm/ttm/ttm_bo_driver.h | 14 ++- 11 files changed, 336 insertions(+), 29 deletions(-)
Change-Id: I34924a40392653e72eeeef143c30ab312cbbf9fa Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/ttm/ttm_bo.c | 23 +++++++++++++++++++++++ include/drm/ttm/ttm_bo_api.h | 1 + include/drm/ttm/ttm_bo_driver.h | 10 ++++++++++ 3 files changed, 34 insertions(+)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 98e06f8bf23b..b740d8f390ca 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1556,6 +1556,7 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, 0x10000000); INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); INIT_LIST_HEAD(&bdev->ddestroy); + INIT_LIST_HEAD(&bdev->process_list); bdev->dev_mapping = mapping; bdev->glob = glob; bdev->need_dma32 = need_dma32; @@ -1569,6 +1570,28 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, } EXPORT_SYMBOL(ttm_bo_device_init);
+int ttm_process_init(struct ttm_process *process, struct ttm_bo_device *bdev, + struct reservation_object *resv) +{ + int i, j; + + INIT_LIST_HEAD(&process->process_list); + for (i = 0; i < TTM_NUM_MEM_TYPES; i++) { + for (j = 0; j < TTM_MAX_BO_PRIORITY; j++) { + INIT_LIST_HEAD(&process->fixed_lru[i][j]); + INIT_LIST_HEAD(&process->dynamic_lru[i][j]); + } + } + spin_lock(&bdev->glob->lru_lock); + list_add_tail(&process->process_list, &bdev->process_list); + spin_unlock(&bdev->glob->lru_lock); + + process->resv = resv; + + return 0; +} +EXPORT_SYMBOL(ttm_process_init); + /* * buffer object vm functions. */ diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index c67977aa1a0e..8cb4b48f387a 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -172,6 +172,7 @@ struct ttm_buffer_object { */
struct ttm_bo_device *bdev; + struct ttm_process *process; enum ttm_bo_type type; void (*destroy) (struct ttm_buffer_object *); unsigned long num_pages; diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 3234cc322e70..91120923de81 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -438,6 +438,13 @@ struct ttm_bo_global {
#define TTM_NUM_MEM_TYPES 8
+struct ttm_process { + struct list_head process_list; + struct list_head fixed_lru[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; + struct list_head dynamic_lru[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; + struct reservation_object *resv; +}; + /** * struct ttm_bo_device - Buffer object driver device-specific data. * @@ -459,6 +466,7 @@ struct ttm_bo_device { * Constant after bo device init / atomic. */ struct list_head device_list; + struct list_head process_list; struct ttm_bo_global *glob; struct ttm_bo_driver *driver; struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES]; @@ -575,6 +583,8 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, struct ttm_bo_global *glob, struct address_space *mapping, uint64_t file_page_offset, bool need_dma32);
+int ttm_process_init(struct ttm_process *process, struct ttm_bo_device *bdev, + struct reservation_object *resv); /** * ttm_bo_unmap_virtual *
Change-Id: I2cf802e641d8b2cdb2bf8bdf1957f3f4f27afaba Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 2 ++ 2 files changed, 4 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index da55a78d7380..6ef449ea8d07 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -2429,6 +2429,8 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, list_add_tail(&vm->root.base.vm_status, &vm->evicted); amdgpu_bo_unreserve(vm->root.base.bo);
+ ttm_process_init(&vm->ttm_vm, &adev->mman.bdev, + vm->root.base.bo->tbo.resv); if (pasid) { unsigned long flags;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 30f080364c97..61b89642fa3d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -29,6 +29,7 @@ #include <linux/rbtree.h> #include <drm/gpu_scheduler.h> #include <drm/drm_file.h> +#include <drm/ttm/ttm_bo_driver.h>
#include "amdgpu_sync.h" #include "amdgpu_ring.h" @@ -166,6 +167,7 @@ struct amdgpu_vm_pt { struct amdgpu_vm { /* tree of virtual addresses mapped */ struct rb_root_cached va; + struct ttm_process ttm_vm;
/* protecting invalidated */ spinlock_t status_lock;
Change-Id: Ie43f3c73cc65526a449208f3ce927b1dfad5cf6b Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 1 + 2 files changed, 3 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 2babfad1fd7f..4b66585a8638 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1629,6 +1629,8 @@ struct amdgpu_device { unsigned long last_mm_index; bool in_gpu_reset; struct mutex lock_reset; + + struct ttm_process kernel_process; };
static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 205da3ff9cd0..4c9e10505e2d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1421,6 +1421,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) return r; } adev->mman.initialized = true; + ttm_process_init(&adev->kernel_process, &adev->mman.bdev, NULL);
/* We opt to avoid OOM on system pages allocations */ adev->mman.bdev.no_retry = true;
Change-Id: I4de8146567b858ae07a8a27cadf71d13d490e8ac Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 7 +++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 28c2706e48d7..37b19ce97699 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -195,6 +195,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, uint64_t size = args->in.bo_size; struct reservation_object *resv = NULL; struct drm_gem_object *gobj; + struct amdgpu_bo *abo; uint32_t handle; int r;
@@ -243,10 +244,12 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, r = amdgpu_gem_object_create(adev, size, args->in.alignment, (u32)(0xffffffff & args->in.domains), flags, false, resv, &gobj); + if (!r) { + abo = gem_to_amdgpu_bo(gobj); + abo->tbo.process = &vm->ttm_vm; + } if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { if (!r) { - struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); - abo->parent = amdgpu_bo_ref(vm->root.base.bo); } amdgpu_bo_unreserve(vm->root.base.bo); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 9e23d6f6f3f3..e911db2d1945 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -447,8 +447,10 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, unsigned long size, else amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
- if (type == ttm_bo_type_kernel) + if (type == ttm_bo_type_kernel) { bo->tbo.priority = 1; + bo->tbo.process = &adev->kernel_process; + }
if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
Change-Id: Id2333f69119222a7e9bdb0357bbed97cf08636da Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/ttm/ttm_bo.c | 59 ++++++++++++++++++++++++++++++++++------- include/drm/ttm/ttm_bo_driver.h | 3 ++- 2 files changed, 52 insertions(+), 10 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index b740d8f390ca..c1d0ec1238c6 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -161,15 +161,20 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; + struct ttm_process *ttm_process = bo->process;
reservation_object_assert_held(bo->resv);
if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { BUG_ON(!list_empty(&bo->lru));
- man = &bdev->man[bo->mem.mem_type]; - list_add_tail(&bo->lru, &man->lru[bo->priority]); kref_get(&bo->list_kref); + if (bo->resv == ttm_process->resv) + list_add_tail(&bo->lru, + &ttm_process->fixed_lru[bo->mem.mem_type][bo->priority]); + else + list_add_tail(&bo->lru, + &ttm_process->dynamic_lru[bo->mem.mem_type][bo->priority]);
if (bo->ttm && !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) { @@ -712,13 +717,35 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, struct ttm_bo_global *glob = bdev->glob; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_buffer_object *bo = NULL; + struct ttm_process *process; bool locked = false; unsigned i; int ret;
spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - list_for_each_entry(bo, &man->lru[i], lru) { + list_for_each_entry(process, &bdev->process_list, process_list) { + list_for_each_entry(bo, &process->dynamic_lru[mem_type][i], lru) { + if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) + continue; + + if (place && !bdev->driver->eviction_valuable(bo, + place)) { + if (locked) + reservation_object_unlock(bo->resv); + continue; + } + break; + } + /* If the inner loop terminated early, we have our candidate */ + if (&bo->lru != &process->dynamic_lru[mem_type][i]) + break; + bo = NULL; + list_for_each_entry(bo, &process->fixed_lru[mem_type][i], lru) { + if (!bo) + continue; + if (&bo->lru == &process->fixed_lru[mem_type][i]) + break; if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) continue;
@@ -732,11 +759,14 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, }
/* If the inner loop terminated early, we have our candidate */ - if (&bo->lru != &man->lru[i]) + if (&bo->lru != &process->fixed_lru[mem_type][i]) break;
bo = NULL; } + if (bo) + break; + }
if (!bo) { spin_unlock(&glob->lru_lock); @@ -1318,13 +1348,13 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - while (!list_empty(&man->lru[i])) { + //while (!list_empty(&man->lru[i])) { spin_unlock(&glob->lru_lock); ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); if (ret) return ret; spin_lock(&glob->lru_lock); - } + //} } spin_unlock(&glob->lru_lock);
@@ -1427,9 +1457,10 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, man->has_type = true; man->use_type = true; man->size = p_size; - +/* for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) INIT_LIST_HEAD(&man->lru[i]); +*/ man->move = NULL;
return 0; @@ -1518,13 +1549,13 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev)
if (ttm_bo_delayed_delete(bdev, true)) pr_debug("Delayed destroy list was clean\n"); - +/* spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) if (list_empty(&bdev->man[0].lru[0])) pr_debug("Swap list %d was clean\n", i); spin_unlock(&glob->lru_lock); - +*/ drm_vma_offset_manager_destroy(&bdev->vma_manager);
return ret; @@ -1591,6 +1622,16 @@ int ttm_process_init(struct ttm_process *process, struct ttm_bo_device *bdev, return 0; } EXPORT_SYMBOL(ttm_process_init); +int ttm_process_fini(struct ttm_process *process, struct ttm_bo_device *bdev) +{ + spin_lock(&bdev->glob->lru_lock); + list_del(&process->process_list); + spin_unlock(&bdev->glob->lru_lock); + + return 0; +} +EXPORT_SYMBOL(ttm_process_fini); +
/* * buffer object vm functions. diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 91120923de81..b6aa7fc5bf14 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -199,7 +199,7 @@ struct ttm_mem_type_manager { * Protected by the global->lru_lock. */
- struct list_head lru[TTM_MAX_BO_PRIORITY]; +// struct list_head lru[TTM_MAX_BO_PRIORITY];
/* * Protected by @move_lock. @@ -585,6 +585,7 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, struct ttm_bo_global *glob,
int ttm_process_init(struct ttm_process *process, struct ttm_bo_device *bdev, struct reservation_object *resv); +int ttm_process_fini(struct ttm_process *process, struct ttm_bo_device *bdev); /** * ttm_bo_unmap_virtual *
Change-Id: Ifb0dc95db6a358cf7f76e2a99f94c58637ad6ee6 Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 17 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 170 ++++++++++++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 6 + drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 1 + 7 files changed, 187 insertions(+), 13 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 4b66585a8638..f398a566f57b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -443,7 +443,8 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, int alignment, u32 initial_domain, u64 flags, enum ttm_bo_type type, struct reservation_object *resv, - struct drm_gem_object **obj); + struct drm_gem_object **obj, + struct ttm_process *process);
int amdgpu_mode_dumb_create(struct drm_file *file_priv, struct drm_device *dev, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c index ff89e84b34ce..1cee2125f570 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c @@ -146,7 +146,7 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev, AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_VRAM_CLEARED, - true, NULL, &gobj); + true, NULL, &gobj, &adev->kernel_process); if (ret) { pr_err("failed to allocate framebuffer (%d)\n", aligned_size); return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 37b19ce97699..d5cbdc810aba 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -45,7 +45,8 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, int alignment, u32 initial_domain, u64 flags, enum ttm_bo_type type, struct reservation_object *resv, - struct drm_gem_object **obj) + struct drm_gem_object **obj, + struct ttm_process *process) { struct amdgpu_bo *bo; int r; @@ -56,8 +57,8 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, alignment = PAGE_SIZE; }
- r = amdgpu_bo_create(adev, size, alignment, initial_domain, - flags, type, resv, &bo); + r = amdgpu_bo_create1(adev, size, alignment, initial_domain, + flags, type, resv, &bo, process); if (r) { DRM_DEBUG("Failed to allocate GEM object (%ld, %d, %u, %d)\n", size, initial_domain, alignment, r); @@ -243,7 +244,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
r = amdgpu_gem_object_create(adev, size, args->in.alignment, (u32)(0xffffffff & args->in.domains), - flags, false, resv, &gobj); + flags, false, resv, &gobj, &vm->ttm_vm); if (!r) { abo = gem_to_amdgpu_bo(gobj); abo->tbo.process = &vm->ttm_vm; @@ -273,6 +274,8 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, { struct ttm_operation_ctx ctx = { true, false }; struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv = filp->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; struct drm_amdgpu_gem_userptr *args = data; struct drm_gem_object *gobj; struct amdgpu_bo *bo; @@ -297,7 +300,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
/* create a gem object to contain this object in */ r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU, - 0, 0, NULL, &gobj); + 0, 0, NULL, &gobj, &vm->ttm_vm); if (r) return r;
@@ -735,6 +738,8 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv, struct drm_mode_create_dumb *args) { struct amdgpu_device *adev = dev->dev_private; + struct amdgpu_fpriv *fpriv = file_priv->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; struct drm_gem_object *gobj; uint32_t handle; int r; @@ -747,7 +752,7 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv, r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_VRAM, AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, - false, NULL, &gobj); + false, NULL, &gobj, &vm->ttm_vm); if (r) return -ENOMEM;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index e911db2d1945..7bb6ee777067 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -421,6 +421,11 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, unsigned long size, bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; #endif
+ if (type == ttm_bo_type_kernel) { + bo->tbo.priority = 1; + bo->tbo.process = &adev->kernel_process; + } + bo->tbo.bdev = &adev->mman.bdev; amdgpu_ttm_placement_from_domain(bo, domains); r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type, @@ -447,11 +452,6 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, unsigned long size, else amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
- if (type == ttm_bo_type_kernel) { - bo->tbo.priority = 1; - bo->tbo.process = &adev->kernel_process; - } - if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { struct dma_fence *fence; @@ -538,6 +538,166 @@ int amdgpu_bo_create(struct amdgpu_device *adev, unsigned long size,
return r; } +static int amdgpu_bo_do_create1(struct amdgpu_device *adev, unsigned long size, + int byte_align, u32 domain, + u64 flags, enum ttm_bo_type type, + struct reservation_object *resv, + struct amdgpu_bo **bo_ptr, + struct ttm_process *process) +{ + struct ttm_operation_ctx ctx = { + .interruptible = (type != ttm_bo_type_kernel), + .no_wait_gpu = false, + .resv = resv, + .flags = TTM_OPT_FLAG_ALLOW_RES_EVICT + }; + struct amdgpu_bo *bo; + unsigned long page_align; + size_t acc_size; + u32 domains, preferred_domains, allowed_domains; + int r; + + page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT; + size = ALIGN(size, PAGE_SIZE); + + if (!amdgpu_bo_validate_size(adev, size, domain)) + return -ENOMEM; + + *bo_ptr = NULL; + + acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, + sizeof(struct amdgpu_bo)); + + preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM | + AMDGPU_GEM_DOMAIN_GTT | + AMDGPU_GEM_DOMAIN_CPU | + AMDGPU_GEM_DOMAIN_GDS | + AMDGPU_GEM_DOMAIN_GWS | + AMDGPU_GEM_DOMAIN_OA); + allowed_domains = preferred_domains; + if (type != ttm_bo_type_kernel && + allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) + allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; + domains = preferred_domains; +retry: + bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); + if (bo == NULL) + return -ENOMEM; + drm_gem_private_object_init(adev->ddev, &bo->gem_base, size); + INIT_LIST_HEAD(&bo->shadow_list); + INIT_LIST_HEAD(&bo->va); + bo->preferred_domains = preferred_domains; + bo->allowed_domains = allowed_domains; + + bo->flags = flags; + +#ifdef CONFIG_X86_32 + /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit + * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 + */ + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; +#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) + /* Don't try to enable write-combining when it can't work, or things + * may be slow + * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 + */ + +#ifndef CONFIG_COMPILE_TEST +#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ + thanks to write-combining +#endif + + if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) + DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " + "better performance thanks to write-combining\n"); + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; +#else + /* For architectures that don't support WC memory, + * mask out the WC flag from the BO + */ + if (!drm_arch_can_wc_memory()) + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; +#endif + + if (type == ttm_bo_type_kernel) + bo->tbo.priority = 1; + bo->tbo.process = process; + + bo->tbo.bdev = &adev->mman.bdev; + amdgpu_ttm_placement_from_domain(bo, domains); + r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type, + &bo->placement, page_align, &ctx, acc_size, + NULL, resv, &amdgpu_ttm_bo_destroy); + if (unlikely(r && r != -ERESTARTSYS) && type == ttm_bo_type_device && + !(flags & AMDGPU_GEM_CREATE_NO_FALLBACK)) { + if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { + flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + goto retry; + } else if (domains != allowed_domains) { + domains = allowed_domains; + goto retry; + } + } + if (unlikely(r)) + return r; + + if (adev->gmc.visible_vram_size < adev->gmc.real_vram_size && + bo->tbo.mem.mem_type == TTM_PL_VRAM && + bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT) + amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, + ctx.bytes_moved); + else + amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); + + if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && + bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { + struct dma_fence *fence; + + r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence); + if (unlikely(r)) + goto fail_unreserve; + + amdgpu_bo_fence(bo, fence, false); + dma_fence_put(bo->tbo.moving); + bo->tbo.moving = dma_fence_get(fence); + dma_fence_put(fence); + } + if (!resv) + amdgpu_bo_unreserve(bo); + *bo_ptr = bo; + + trace_amdgpu_bo_create(bo); + + /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ + if (type == ttm_bo_type_device) + bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + + return 0; + +fail_unreserve: + if (!resv) + ww_mutex_unlock(&bo->tbo.resv->lock); + amdgpu_bo_unref(&bo); + return r; +} + + +int amdgpu_bo_create1(struct amdgpu_device *adev, unsigned long size, + int byte_align, u32 domain, + u64 flags, enum ttm_bo_type type, + struct reservation_object *resv, + struct amdgpu_bo **bo_ptr, struct ttm_process *process) +{ + uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW; + int r; + + r = amdgpu_bo_do_create1(adev, size, byte_align, domain, + parent_flags, type, resv, bo_ptr, process); + if (r) + return r; + return r; +} +
int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev, struct amdgpu_ring *ring, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 546f77cb7882..d4191f0424d1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -208,6 +208,12 @@ int amdgpu_bo_create(struct amdgpu_device *adev, unsigned long size, u64 flags, enum ttm_bo_type type, struct reservation_object *resv, struct amdgpu_bo **bo_ptr); +int amdgpu_bo_create1(struct amdgpu_device *adev, unsigned long size, + int byte_align, u32 domain, + u64 flags, enum ttm_bo_type type, + struct reservation_object *resv, + struct amdgpu_bo **bo_ptr, struct ttm_process *process); + int amdgpu_bo_create_reserved(struct amdgpu_device *adev, unsigned long size, int align, u32 domain, struct amdgpu_bo **bo_ptr, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 4c9e10505e2d..591db2df1685 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1535,6 +1535,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) return;
amdgpu_ttm_debugfs_fini(adev); + ttm_process_fini(&adev->kernel_process, &adev->mman.bdev); amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL); amdgpu_ttm_fw_reserve_vram_fini(adev); if (adev->mman.aper_base_kaddr) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 6ef449ea8d07..3a54b236fd42 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -2612,6 +2612,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) amdgpu_vm_free_mapping(adev, vm, mapping, NULL); }
+ ttm_process_fini(&vm->ttm_vm, &adev->mman.bdev); root = amdgpu_bo_ref(vm->root.base.bo); r = amdgpu_bo_reserve(root, true); if (r) {
Change-Id: I0f533c6512f3b72fcf2fbf11d738f38d9e087f26 Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/ttm/ttm_bo.c | 39 +++++++++++++++++++++++++++++++-------- include/drm/ttm/ttm_bo_api.h | 3 ++- include/drm/ttm/ttm_bo_driver.h | 2 +- 3 files changed, 34 insertions(+), 10 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index c1d0ec1238c6..73343d1108d2 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -157,6 +157,26 @@ static void ttm_bo_release_list(struct kref *list_kref) ttm_mem_global_free(bdev->glob->mem_glob, acc_size); }
+static void ttm_bo_add_to_rb(struct ttm_buffer_object *bo, + struct rb_root *root) { + struct rb_node **new = &(root->rb_node), *parent = NULL; + + while (*new) { + struct ttm_buffer_object *this = + container_of(*new, struct ttm_buffer_object, node); + int result = bo->index - this->index; + + parent = *new; + if (result < 0) + new = &((*new)->rb_left); + else if (result > 0) + new = &((*new)->rb_right); + else + return; + } + rb_link_node(&bo->node, parent, new); + rb_insert_color(&bo->node, root); +} void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; @@ -170,7 +190,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
kref_get(&bo->list_kref); if (bo->resv == ttm_process->resv) - list_add_tail(&bo->lru, + ttm_bo_add_to_rb(bo, &ttm_process->fixed_lru[bo->mem.mem_type][bo->priority]); else list_add_tail(&bo->lru, @@ -200,8 +220,12 @@ void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) if (!list_empty(&bo->lru)) { list_del_init(&bo->lru); kref_put(&bo->list_kref, ttm_bo_ref_bug); + } else if (RB_EMPTY_NODE(&bo->node)) { + rb_erase(&bo->node, + &bo->process->fixed_lru[bo->mem.mem_type][bo->priority]); + RB_CLEAR_NODE(&bo->node); + kref_put(&bo->list_kref, ttm_bo_ref_bug); } - /* * TODO: Add a driver hook to delete from * driver-specific LRU's here. @@ -725,6 +749,7 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { list_for_each_entry(process, &bdev->process_list, process_list) { + struct rb_node *node; list_for_each_entry(bo, &process->dynamic_lru[mem_type][i], lru) { if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) continue; @@ -741,11 +766,9 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, if (&bo->lru != &process->dynamic_lru[mem_type][i]) break; bo = NULL; - list_for_each_entry(bo, &process->fixed_lru[mem_type][i], lru) { - if (!bo) - continue; - if (&bo->lru == &process->fixed_lru[mem_type][i]) - break; + for (node = rb_first(&process->fixed_lru[mem_type][i]); node; + node = rb_next(node)) { + bo = rb_entry(node, struct ttm_buffer_object, node); if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) continue;
@@ -1609,7 +1632,7 @@ int ttm_process_init(struct ttm_process *process, struct ttm_bo_device *bdev, INIT_LIST_HEAD(&process->process_list); for (i = 0; i < TTM_NUM_MEM_TYPES; i++) { for (j = 0; j < TTM_MAX_BO_PRIORITY; j++) { - INIT_LIST_HEAD(&process->fixed_lru[i][j]); + process->fixed_lru[i][j] = RB_ROOT; INIT_LIST_HEAD(&process->dynamic_lru[i][j]); } } diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index 8cb4b48f387a..9c8179bdd685 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -170,7 +170,7 @@ struct ttm_buffer_object { /** * Members constant at init. */ - + struct rb_node node; struct ttm_bo_device *bdev; struct ttm_process *process; enum ttm_bo_type type; @@ -232,6 +232,7 @@ struct ttm_buffer_object { struct reservation_object *resv; struct reservation_object ttm_resv; struct mutex wu_mutex; + u64 index; };
/** diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index b6aa7fc5bf14..818aee15d1ec 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -440,7 +440,7 @@ struct ttm_bo_global {
struct ttm_process { struct list_head process_list; - struct list_head fixed_lru[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; + struct rb_root fixed_lru[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; struct list_head dynamic_lru[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; struct reservation_object *resv; };
Change-Id: I4abf5cf0aaf946162dabd08fc1fd0406c2abf418 Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/ttm/ttm_bo.c | 1 + include/drm/ttm/ttm_bo_driver.h | 1 + 2 files changed, 2 insertions(+)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 73343d1108d2..d56312702b49 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1641,6 +1641,7 @@ int ttm_process_init(struct ttm_process *process, struct ttm_bo_device *bdev, spin_unlock(&bdev->glob->lru_lock);
process->resv = resv; + atomic64_set(&process->bo_index, 0);
return 0; } diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 818aee15d1ec..4c42df6afcfe 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -443,6 +443,7 @@ struct ttm_process { struct rb_root fixed_lru[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; struct list_head dynamic_lru[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; struct reservation_object *resv; + atomic64_t bo_index; };
/**
Change-Id: I877b2d5c54b3e47b66f2d58454dcf1e5f5a68972 Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 2 ++ 1 file changed, 2 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 7bb6ee777067..d54abba4e017 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -622,8 +622,10 @@ static int amdgpu_bo_do_create1(struct amdgpu_device *adev, unsigned long size, if (type == ttm_bo_type_kernel) bo->tbo.priority = 1; bo->tbo.process = process; + bo->tbo.index = (u64)atomic64_inc_return(&process->bo_index);
bo->tbo.bdev = &adev->mman.bdev; + RB_CLEAR_NODE(&bo->tbo.node); amdgpu_ttm_placement_from_domain(bo, domains); r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type, &bo->placement, page_align, &ctx, acc_size,
Change-Id: Ib68bff91fd127162cf8c72516101546e1fe014df Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 1 - drivers/gpu/drm/ttm/ttm_bo.c | 39 ++++++++++++++++++++---------- 2 files changed, 26 insertions(+), 14 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index d54abba4e017..ea95c2e9a858 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -625,7 +625,6 @@ static int amdgpu_bo_do_create1(struct amdgpu_device *adev, unsigned long size, bo->tbo.index = (u64)atomic64_inc_return(&process->bo_index);
bo->tbo.bdev = &adev->mman.bdev; - RB_CLEAR_NODE(&bo->tbo.node); amdgpu_ttm_placement_from_domain(bo, domains); r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type, &bo->placement, page_align, &ctx, acc_size, diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index d56312702b49..dc9545eeb5d6 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -164,15 +164,12 @@ static void ttm_bo_add_to_rb(struct ttm_buffer_object *bo, while (*new) { struct ttm_buffer_object *this = container_of(*new, struct ttm_buffer_object, node); - int result = bo->index - this->index;
parent = *new; - if (result < 0) + if (bo->index < this->index) new = &((*new)->rb_left); - else if (result > 0) - new = &((*new)->rb_right); else - return; + new = &((*new)->rb_right); } rb_link_node(&bo->node, parent, new); rb_insert_color(&bo->node, root); @@ -211,6 +208,24 @@ static void ttm_bo_ref_bug(struct kref *list_kref) BUG(); }
+static struct ttm_buffer_object *ttm_bo_rb_find(struct rb_root *root, u64 index) +{ + struct rb_node *node = root->rb_node; + + while (node) { + struct ttm_buffer_object *bo = + container_of(node, struct ttm_buffer_object, node); + + if (index < bo->index) + node = node->rb_left; + else if (index > bo->index) + node = node->rb_right; + else + return bo; + } + + return NULL; +} void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) { if (!list_empty(&bo->swap)) { @@ -220,10 +235,10 @@ void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) if (!list_empty(&bo->lru)) { list_del_init(&bo->lru); kref_put(&bo->list_kref, ttm_bo_ref_bug); - } else if (RB_EMPTY_NODE(&bo->node)) { + } else if + (ttm_bo_rb_find(&bo->process->fixed_lru[bo->mem.mem_type][bo->priority], bo->index)) { rb_erase(&bo->node, &bo->process->fixed_lru[bo->mem.mem_type][bo->priority]); - RB_CLEAR_NODE(&bo->node); kref_put(&bo->list_kref, ttm_bo_ref_bug); } /* @@ -769,23 +784,21 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, for (node = rb_first(&process->fixed_lru[mem_type][i]); node; node = rb_next(node)) { bo = rb_entry(node, struct ttm_buffer_object, node); - if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) + if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) { + bo = NULL; continue; + }
if (place && !bdev->driver->eviction_valuable(bo, place)) { if (locked) reservation_object_unlock(bo->resv); + bo = NULL; continue; } break; }
- /* If the inner loop terminated early, we have our candidate */ - if (&bo->lru != &process->fixed_lru[mem_type][i]) - break; - - bo = NULL; } if (bo) break;
Change-Id: I5b6afbdd715e28e5266b5099ca9a34399d1fc3a1 Signed-off-by: Chunming Zhou david1.zhou@amd.com --- drivers/gpu/drm/ttm/ttm_bo.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index dc9545eeb5d6..d6e7c835f4c1 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -261,8 +261,13 @@ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { reservation_object_assert_held(bo->resv);
- ttm_bo_del_from_lru(bo); - ttm_bo_add_to_lru(bo); + if (bo->resv == bo->process->resv) { + list_move_tail(&bo->process->process_list, + &bo->bdev->process_list); + } else { + ttm_bo_del_from_lru(bo); + ttm_bo_add_to_lru(bo); + } } EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
dri-devel@lists.freedesktop.org
-
Chunming Zhou