From: Jerome Glisse jglisse@redhat.com
This allow to associate a fence with sa bo and retry and wait if sa bo alloc can block.
v2: bug fixes
Signed-off-by: Jerome Glisse jglisse@redhat.com --- drivers/gpu/drm/radeon/radeon.h | 10 ++- drivers/gpu/drm/radeon/radeon_cs.c | 4 +- drivers/gpu/drm/radeon/radeon_gart.c | 14 ++-- drivers/gpu/drm/radeon/radeon_object.h | 10 ++-- drivers/gpu/drm/radeon/radeon_ring.c | 14 ++-- drivers/gpu/drm/radeon/radeon_sa.c | 102 ++++++++++++++++++++++++++--- drivers/gpu/drm/radeon/radeon_semaphore.c | 4 +- 7 files changed, 122 insertions(+), 36 deletions(-)
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 59bcfb9..4815ebe 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -379,6 +379,8 @@ struct radeon_bo_list { * Assumption is that there won't be hole (all object on same * alignment). */ +struct radeon_sa_bo; + struct radeon_sa_manager { spinlock_t lock; struct radeon_bo *bo; @@ -390,8 +392,6 @@ struct radeon_sa_manager { uint32_t domain; };
-struct radeon_sa_bo; - /* sub-allocation buffer */ struct radeon_sa_bo { struct list_head list; @@ -399,6 +399,8 @@ struct radeon_sa_bo { unsigned soffset; unsigned eoffset; unsigned size; + struct radeon_fence *fence; + bool free; };
/* @@ -626,7 +628,7 @@ void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc); */
struct radeon_ib { - struct radeon_sa_bo sa_bo; + struct radeon_sa_bo *sa_bo; unsigned idx; uint32_t length_dw; uint64_t gpu_addr; @@ -680,7 +682,7 @@ struct radeon_vm { unsigned last_pfn; u64 pt_gpu_addr; u64 *pt; - struct radeon_sa_bo sa_bo; + struct radeon_sa_bo *sa_bo; struct mutex mutex; /* last fence for cs using this vm */ struct radeon_fence *fence; diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index 8de6b3a..b39f22e 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c @@ -476,7 +476,7 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, /* ib pool is bind at 0 in virtual address space to gpu_addr is the * offset inside the pool bo */ - parser->const_ib->gpu_addr = parser->const_ib->sa_bo.soffset; + parser->const_ib->gpu_addr = parser->const_ib->sa_bo->soffset; r = radeon_ib_schedule(rdev, parser->const_ib); if (r) goto out; @@ -486,7 +486,7 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, /* ib pool is bind at 0 in virtual address space to gpu_addr is the * offset inside the pool bo */ - parser->ib->gpu_addr = parser->ib->sa_bo.soffset; + parser->ib->gpu_addr = parser->ib->sa_bo->soffset; parser->ib->is_const_ib = false; r = radeon_ib_schedule(rdev, parser->ib); out: diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c index 4a5d9d4..89328e3 100644 --- a/drivers/gpu/drm/radeon/radeon_gart.c +++ b/drivers/gpu/drm/radeon/radeon_gart.c @@ -393,19 +393,19 @@ int radeon_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm) }
retry: - r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, &vm->sa_bo, - RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8), - RADEON_GPU_PAGE_SIZE); - if (r) { + vm->sa_bo = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, + RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8), + RADEON_GPU_PAGE_SIZE, false, NULL); + if (vm->sa_bo == NULL) { if (list_empty(&rdev->vm_manager.lru_vm)) { - return r; + return -ENOMEM; } vm_evict = list_first_entry(&rdev->vm_manager.lru_vm, struct radeon_vm, list); radeon_vm_unbind(rdev, vm_evict); goto retry; } - vm->pt = radeon_sa_bo_cpu_addr(&vm->sa_bo); - vm->pt_gpu_addr = radeon_sa_bo_gpu_addr(&vm->sa_bo); + vm->pt = radeon_sa_bo_cpu_addr(vm->sa_bo); + vm->pt_gpu_addr = radeon_sa_bo_gpu_addr(vm->sa_bo); memset(vm->pt, 0, RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8));
retry_id: diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h index 99ab46a..7bbc319 100644 --- a/drivers/gpu/drm/radeon/radeon_object.h +++ b/drivers/gpu/drm/radeon/radeon_object.h @@ -166,12 +166,12 @@ extern int radeon_sa_bo_manager_start(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager); extern int radeon_sa_bo_manager_suspend(struct radeon_device *rdev, struct radeon_sa_manager *sa_manager); -extern int radeon_sa_bo_new(struct radeon_device *rdev, - struct radeon_sa_manager *sa_manager, - struct radeon_sa_bo *sa_bo, - unsigned size, unsigned align); +extern struct radeon_sa_bo *radeon_sa_bo_new(struct radeon_device *rdev, + struct radeon_sa_manager *sa_manager, + unsigned size, unsigned align, + bool block, struct radeon_fence *fence); extern void radeon_sa_bo_free(struct radeon_device *rdev, - struct radeon_sa_bo *sa_bo); + struct radeon_sa_bo **sa_bo); #if defined(CONFIG_DEBUG_FS) extern void radeon_sa_bo_dump_debug_info(struct radeon_sa_manager *sa_manager, struct seq_file *m); diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c index 981ab95..b646bdb 100644 --- a/drivers/gpu/drm/radeon/radeon_ring.c +++ b/drivers/gpu/drm/radeon/radeon_ring.c @@ -122,13 +122,12 @@ retry: for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { radeon_ib_try_free(rdev, &rdev->ib_pool.ibs[idx]); if (rdev->ib_pool.ibs[idx].fence == NULL) { - r = radeon_sa_bo_new(rdev, &rdev->ib_pool.sa_manager, - &rdev->ib_pool.ibs[idx].sa_bo, - size, 256); - if (!r) { + rdev->ib_pool.ibs[idx].sa_bo = radeon_sa_bo_new(rdev, &rdev->ib_pool.sa_manager, + size, 256, false, NULL); + if (rdev->ib_pool.ibs[idx].sa_bo) { *ib = &rdev->ib_pool.ibs[idx]; - (*ib)->ptr = radeon_sa_bo_cpu_addr(&(*ib)->sa_bo); - (*ib)->gpu_addr = radeon_sa_bo_gpu_addr(&(*ib)->sa_bo); + (*ib)->ptr = radeon_sa_bo_cpu_addr((*ib)->sa_bo); + (*ib)->gpu_addr = radeon_sa_bo_gpu_addr((*ib)->sa_bo); (*ib)->fence = fence; (*ib)->vm_id = 0; (*ib)->is_const_ib = false; @@ -146,6 +145,7 @@ retry: } /* this should be rare event, ie all ib scheduled none signaled yet. */ + r = -ENOMEM; for (i = 0; i < RADEON_IB_POOL_SIZE; i++) { if (rdev->ib_pool.ibs[idx].fence && rdev->ib_pool.ibs[idx].fence->emitted) { r = radeon_fence_wait(rdev->ib_pool.ibs[idx].fence, false); @@ -226,7 +226,7 @@ int radeon_ib_pool_init(struct radeon_device *rdev) rdev->ib_pool.ibs[i].fence = NULL; rdev->ib_pool.ibs[i].idx = i; rdev->ib_pool.ibs[i].length_dw = 0; - INIT_LIST_HEAD(&rdev->ib_pool.ibs[i].sa_bo.list); + rdev->ib_pool.ibs[i].sa_bo = NULL; } rdev->ib_pool.head_id = 0; rdev->ib_pool.ready = true; diff --git a/drivers/gpu/drm/radeon/radeon_sa.c b/drivers/gpu/drm/radeon/radeon_sa.c index 63b0cd2..d7d7b7e 100644 --- a/drivers/gpu/drm/radeon/radeon_sa.c +++ b/drivers/gpu/drm/radeon/radeon_sa.c @@ -122,6 +122,12 @@ static void radeon_sa_bo_free_locked(struct radeon_device *rdev, struct radeon_s struct radeon_sa_manager *sa_manager = sa_bo->manager; struct list_head *prev;
+ if (sa_bo->fence) { + if (!radeon_fence_signaled(sa_bo->fence)) { + return; + } + radeon_fence_unref(&sa_bo->fence); + } prev = sa_bo->list.prev; list_del_init(&sa_bo->list); if (list_empty(&sa_manager->sa_bo)) { @@ -138,6 +144,25 @@ static void radeon_sa_bo_free_locked(struct radeon_device *rdev, struct radeon_s sa_manager->last = list_entry(prev, struct radeon_sa_bo, list); } } + /* in case try free already free the sa_bo but radeon_sa_bo_free + * wasn't yet call, the free bool protect us from freeing to + * early the structure + */ + if (sa_bo->free) { + kfree(sa_bo); + } +} + +static bool radeon_sa_manager_try_free(struct radeon_device *rdev, + struct radeon_sa_bo *oldest) +{ + if (oldest->fence && oldest->fence->emitted) { + if (radeon_fence_signaled(oldest->fence)) { + radeon_sa_bo_free_locked(rdev, oldest); + return true; + } + } + return false; }
/* @@ -151,25 +176,32 @@ static void radeon_sa_bo_free_locked(struct radeon_device *rdev, struct radeon_s * * Alignment can't be bigger than page size */ -int radeon_sa_bo_new(struct radeon_device *rdev, - struct radeon_sa_manager *sa_manager, - struct radeon_sa_bo *sa_bo, - unsigned size, unsigned align) +struct radeon_sa_bo *radeon_sa_bo_new(struct radeon_device *rdev, + struct radeon_sa_manager *sa_manager, + unsigned size, unsigned align, + bool block, struct radeon_fence *fence) { - struct radeon_sa_bo *next, *oldest; + struct radeon_sa_bo *sa_bo, *next, *oldest; unsigned offset, wasted, hole_offset, hole_size; bool try_begining = false, add_begining = false;
BUG_ON(align > RADEON_GPU_PAGE_SIZE); BUG_ON(size > sa_manager->size);
+ sa_bo = kmalloc(sizeof(struct radeon_sa_bo), GFP_KERNEL); + if (sa_bo == NULL) { + return NULL; + } sa_bo->manager = sa_manager; + sa_bo->fence = NULL; + sa_bo->free = false; sa_bo->soffset = 0; sa_bo->eoffset = 0; sa_bo->size = 0; INIT_LIST_HEAD(&sa_bo->list);
spin_lock(&sa_manager->lock); +retry: if (sa_manager->last == NULL) { offset = 0; add_begining = true; @@ -186,6 +218,7 @@ int radeon_sa_bo_new(struct radeon_device *rdev, } else { next = list_entry(sa_manager->last->list.next, struct radeon_sa_bo, list); hole_size = next->soffset - hole_offset; + oldest = next; } if ((size + wasted) >= hole_size) { offset = hole_offset + wasted; @@ -201,9 +234,44 @@ int radeon_sa_bo_new(struct radeon_device *rdev, goto out; } } + /* try to be optimist and free the oldest one */ + if (radeon_sa_manager_try_free(rdev, oldest)) { + goto retry; + } + + /* if block is used all the sa_bo must be associated with a + * fence, we perform sanity check but expect things to go + * berserk if you don't follow this + */ + if (block) { + struct radeon_fence *fence = NULL; + int r; + + if (oldest->fence) { + fence = radeon_fence_ref(oldest->fence); + } + spin_unlock(&sa_manager->lock); + + if (fence == NULL) { + /* this should never happen */ + dev_warn(rdev->dev, "sa allocator nothing we can wait for\n"); + goto out_err; + }
+ r = radeon_fence_wait(fence, false); + radeon_fence_unref(&fence); + if (r) { + goto out_err; + } + + spin_lock(&sa_manager->lock); + goto retry; + } spin_unlock(&sa_manager->lock); - return -ENOMEM; + +out_err: + kfree(sa_bo); + return NULL;
out: if (add_begining) { @@ -212,22 +280,38 @@ out: list_add(&sa_bo->list, &sa_manager->last->list); } sa_manager->last = sa_bo; + if (fence) { + sa_bo->fence = radeon_fence_ref(fence); + } sa_bo->soffset = offset; sa_bo->eoffset = offset + size; sa_bo->size = size; spin_unlock(&sa_manager->lock); - return 0; + return sa_bo; }
-void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo *sa_bo) +void radeon_sa_bo_free(struct radeon_device *rdev, struct radeon_sa_bo **tmp) { - struct radeon_sa_manager *sa_manager = sa_bo->manager; + struct radeon_sa_bo *sa_bo; + struct radeon_sa_manager *sa_manager;
+ if (tmp == NULL || *tmp == NULL) { + return; + } + + sa_bo = *tmp; + sa_manager = sa_bo->manager; + *tmp = NULL; spin_lock(&sa_manager->lock); + sa_bo->free = true; if (list_empty(&sa_bo->list)) { /* it has already been free */ + kfree(sa_bo); goto out; } + if (sa_bo->fence && !sa_bo->fence->emitted) { + radeon_fence_unref(&sa_bo->fence); + } radeon_sa_bo_free_locked(rdev, sa_bo);
out: diff --git a/drivers/gpu/drm/radeon/radeon_semaphore.c b/drivers/gpu/drm/radeon/radeon_semaphore.c index c3763e4..d79afb3 100644 --- a/drivers/gpu/drm/radeon/radeon_semaphore.c +++ b/drivers/gpu/drm/radeon/radeon_semaphore.c @@ -55,9 +55,9 @@ static int radeon_semaphore_add_bo(struct radeon_device *rdev) return r; } gpu_addr = rdev->ib_pool.sa_manager.gpu_addr; - gpu_addr += bo->ib->sa_bo.soffset; + gpu_addr += bo->ib->sa_bo->soffset; cpu_ptr = rdev->ib_pool.sa_manager.cpu_ptr; - cpu_ptr += (bo->ib->sa_bo.soffset >> 2); + cpu_ptr += (bo->ib->sa_bo->soffset >> 2); for (i = 0; i < (RADEON_SEMAPHORE_BO_SIZE/8); i++) { bo->semaphores[i].gpu_addr = gpu_addr; bo->semaphores[i].cpu_ptr = cpu_ptr;