It seems some drivers really want this as a parameter, like vmwgfx.

Signed-off-by: Maarten Lankhorst maarten.lankhorst@canonical.com --- drivers/gpu/drm/qxl/qxl_release.c | 2 +- drivers/gpu/drm/radeon/radeon_object.c | 2 +- drivers/gpu/drm/radeon/radeon_uvd.c | 2 +- drivers/gpu/drm/radeon/radeon_vm.c | 2 +- drivers/gpu/drm/ttm/ttm_execbuf_util.c | 22 +++++++++++++--------- drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c | 7 ++----- drivers/gpu/drm/vmwgfx/vmwgfx_resource.c | 2 +- include/drm/ttm/ttm_execbuf_util.h | 9 +++++---- 8 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c index 14e776f1d14e..2b43e5deb051 100644 --- a/drivers/gpu/drm/qxl/qxl_release.c +++ b/drivers/gpu/drm/qxl/qxl_release.c @@ -159,7 +159,7 @@ int qxl_release_reserve_list(struct qxl_release *release, bool no_intr) if (list_is_singular(&release->bos)) return 0;

- ret = ttm_eu_reserve_buffers(&release->ticket, &release->bos); + ret = ttm_eu_reserve_buffers(&release->ticket, &release->bos, !no_intr); if (ret) return ret;

diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c index 6c717b257d6d..a3ed725ea641 100644 --- a/drivers/gpu/drm/radeon/radeon_object.c +++ b/drivers/gpu/drm/radeon/radeon_object.c @@ -438,7 +438,7 @@ int radeon_bo_list_validate(struct radeon_device *rdev, u64 bytes_moved = 0, initial_bytes_moved; u64 bytes_moved_threshold = radeon_bo_get_threshold_for_moves(rdev);

- r = ttm_eu_reserve_buffers(ticket, head); + r = ttm_eu_reserve_buffers(ticket, head, true); if (unlikely(r != 0)) { return r; } diff --git a/drivers/gpu/drm/radeon/radeon_uvd.c b/drivers/gpu/drm/radeon/radeon_uvd.c index a4ad270e8261..67b2a367df40 100644 --- a/drivers/gpu/drm/radeon/radeon_uvd.c +++ b/drivers/gpu/drm/radeon/radeon_uvd.c @@ -620,7 +620,7 @@ static int radeon_uvd_send_msg(struct radeon_device *rdev, INIT_LIST_HEAD(&head); list_add(&tv.head, &head);

- r = ttm_eu_reserve_buffers(&ticket, &head); + r = ttm_eu_reserve_buffers(&ticket, &head, true); if (r) return r;

diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index 725d3669014f..0a0d0291af0d 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -365,7 +365,7 @@ static int radeon_vm_clear_bo(struct radeon_device *rdev, INIT_LIST_HEAD(&head); list_add(&tv.head, &head);

- r = ttm_eu_reserve_buffers(&ticket, &head); + r = ttm_eu_reserve_buffers(&ticket, &head, true); if (r) return r;

diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index e8dac8758528..39a11bbd2bac 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -112,7 +112,7 @@ EXPORT_SYMBOL(ttm_eu_backoff_reservation); */

int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, - struct list_head *list) + struct list_head *list, bool intr) { struct ttm_bo_global *glob; struct ttm_validate_buffer *entry; @@ -140,7 +140,7 @@ retry: if (entry->reserved) continue;

- ret = __ttm_bo_reserve(bo, true, (ticket == NULL), true, + ret = __ttm_bo_reserve(bo, intr, (ticket == NULL), true, ticket);

if (ret == -EDEADLK) { @@ -153,13 +153,17 @@ retry: ttm_eu_backoff_reservation_locked(list); spin_unlock(&glob->lru_lock); ttm_eu_list_ref_sub(list); - ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, - ticket); - if (unlikely(ret != 0)) { - if (ret == -EINTR) - ret = -ERESTARTSYS; - goto err_fini; - } + + if (intr) { + ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock, + ticket); + if (unlikely(ret != 0)) { + if (ret == -EINTR) + ret = -ERESTARTSYS; + goto err_fini; + } + } else + ww_mutex_lock_slow(&bo->resv->lock, ticket);

entry->reserved = true; if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index 7bfdaa163a33..24f067bf438d 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -2496,7 +2496,7 @@ int vmw_execbuf_process(struct drm_file *file_priv, if (unlikely(ret != 0)) goto out_err_nores;

- ret = ttm_eu_reserve_buffers(&ticket, &sw_context->validate_nodes); + ret = ttm_eu_reserve_buffers(&ticket, &sw_context->validate_nodes, true); if (unlikely(ret != 0)) goto out_err;

@@ -2684,10 +2684,7 @@ void __vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv, query_val.bo = ttm_bo_reference(dev_priv->dummy_query_bo); list_add_tail(&query_val.head, &validate_list);

- do { - ret = ttm_eu_reserve_buffers(&ticket, &validate_list); - } while (ret == -ERESTARTSYS); - + ret = ttm_eu_reserve_buffers(&ticket, &validate_list, false); if (unlikely(ret != 0)) { vmw_execbuf_unpin_panic(dev_priv); goto out_no_reserve; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c index a432c0db257c..9d8c0a3f391c 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c @@ -1215,7 +1215,7 @@ vmw_resource_check_buffer(struct vmw_resource *res, INIT_LIST_HEAD(&val_list); val_buf->bo = ttm_bo_reference(&res->backup->base); list_add_tail(&val_buf->head, &val_list); - ret = ttm_eu_reserve_buffers(NULL, &val_list); + ret = ttm_eu_reserve_buffers(NULL, &val_list, interruptible); if (unlikely(ret != 0)) goto out_no_reserve;

diff --git a/include/drm/ttm/ttm_execbuf_util.h b/include/drm/ttm/ttm_execbuf_util.h index 16db7d01a336..fd95fd569ca3 100644 --- a/include/drm/ttm/ttm_execbuf_util.h +++ b/include/drm/ttm/ttm_execbuf_util.h @@ -73,6 +73,7 @@ extern void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, * @ticket: [out] ww_acquire_ctx filled in by call, or NULL if only * non-blocking reserves should be tried. * @list: thread private list of ttm_validate_buffer structs. + * @intr: should the wait be interruptible * * Tries to reserve bos pointed to by the list entries for validation. * If the function returns 0, all buffers are marked as "unfenced", @@ -84,9 +85,9 @@ extern void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, * CPU write reservations to be cleared, and for other threads to * unreserve their buffers. * - * This function may return -ERESTART or -EAGAIN if the calling process - * receives a signal while waiting. In that case, no buffers on the list - * will be reserved upon return. + * If intr is set to true, this function may return -ERESTARTSYS if the + * calling process receives a signal while waiting. In that case, no + * buffers on the list will be reserved upon return. * * Buffers reserved by this function should be unreserved by * a call to either ttm_eu_backoff_reservation() or @@ -95,7 +96,7 @@ extern void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, */

extern int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, - struct list_head *list); + struct list_head *list, bool intr);

/** * function ttm_eu_fence_buffer_objects.

Apart from some code inside ttm itself and nouveau_bo_vma_del, this is the only place where ttm_bo_wait is used without a reservation. Fix this so we can remove the fence_lock later on.

After the switch to rcu the reservation lock will be removed again.

Signed-off-by: Maarten Lankhorst maarten.lankhorst@canonical.com --- drivers/gpu/drm/nouveau/nouveau_gem.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index df9d451afdcd..0da3740f150f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -886,17 +886,31 @@ nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, struct drm_gem_object *gem; struct nouveau_bo *nvbo; bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); - int ret = -EINVAL; + int ret; + struct nouveau_fence *fence = NULL;

gem = drm_gem_object_lookup(dev, file_priv, req->handle); if (!gem) return -ENOENT; nvbo = nouveau_gem_object(gem);

- spin_lock(&nvbo->bo.bdev->fence_lock); - ret = ttm_bo_wait(&nvbo->bo, true, true, no_wait); - spin_unlock(&nvbo->bo.bdev->fence_lock); + ret = ttm_bo_reserve(&nvbo->bo, true, false, false, 0); + if (!ret) { + spin_lock(&nvbo->bo.bdev->fence_lock); + ret = ttm_bo_wait(&nvbo->bo, true, true, true); + if (!no_wait && ret) + fence = nouveau_fence_ref(nvbo->bo.sync_obj); + spin_unlock(&nvbo->bo.bdev->fence_lock); + + ttm_bo_unreserve(&nvbo->bo); + } drm_gem_object_unreference_unlocked(gem); + + if (fence) { + ret = nouveau_fence_wait(fence, true, no_wait); + nouveau_fence_unref(&fence); + } + return ret; }

This is the last remaining function that doesn't use the reservation lock completely to fence off access to a buffer. --- drivers/gpu/drm/ttm/ttm_bo.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 3da89d5dab60..df08f18f05ac 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -502,17 +502,6 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, if (ret) return ret;

- /* - * remove sync_obj with ttm_bo_wait, the wait should be - * finished, and no new wait object should have been added. - */ - spin_lock(&bdev->fence_lock); - ret = ttm_bo_wait(bo, false, false, true); - WARN_ON(ret); - spin_unlock(&bdev->fence_lock); - if (ret) - return ret; - spin_lock(&glob->lru_lock); ret = __ttm_bo_reserve(bo, false, true, false, NULL);

@@ -528,8 +517,16 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, spin_unlock(&glob->lru_lock); return 0; } - } else - spin_unlock(&bdev->fence_lock); + + /* + * remove sync_obj with ttm_bo_wait, the wait should be + * finished, and no new wait object should have been added. + */ + spin_lock(&bdev->fence_lock); + ret = ttm_bo_wait(bo, false, false, true); + WARN_ON(ret); + } + spin_unlock(&bdev->fence_lock);

if (ret || unlikely(list_empty(&bo->ddestroy))) { __ttm_bo_unreserve(bo); @@ -1539,6 +1536,8 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, void *sync_obj; int ret = 0;

+ lockdep_assert_held(&bo->resv->lock.base); + if (likely(bo->sync_obj == NULL)) return 0;

From: Maarten Lankhorst maarten.lankhorst@ubuntu.com

Signed-off-by: Maarten Lankhorst maarten.lankhorst@canonical.com --- drivers/gpu/drm/nouveau/core/core/event.c | 4 drivers/gpu/drm/nouveau/nouveau_bo.c | 6 drivers/gpu/drm/nouveau/nouveau_display.c | 4 drivers/gpu/drm/nouveau/nouveau_fence.c | 435 ++++++++++++++++++++--------- drivers/gpu/drm/nouveau/nouveau_fence.h | 20 + drivers/gpu/drm/nouveau/nouveau_gem.c | 17 - drivers/gpu/drm/nouveau/nv04_fence.c | 4 drivers/gpu/drm/nouveau/nv10_fence.c | 4 drivers/gpu/drm/nouveau/nv17_fence.c | 2 drivers/gpu/drm/nouveau/nv50_fence.c | 2 drivers/gpu/drm/nouveau/nv84_fence.c | 11 - 11 files changed, 330 insertions(+), 179 deletions(-)

diff --git a/drivers/gpu/drm/nouveau/core/core/event.c b/drivers/gpu/drm/nouveau/core/core/event.c index ae81d3b5d8b7..5ddc28ec7660 100644 --- a/drivers/gpu/drm/nouveau/core/core/event.c +++ b/drivers/gpu/drm/nouveau/core/core/event.c @@ -139,14 +139,14 @@ nouveau_event_ref(struct nouveau_eventh *handler, struct nouveau_eventh **ref) void nouveau_event_trigger(struct nouveau_event *event, u32 types, int index) { - struct nouveau_eventh *handler; + struct nouveau_eventh *handler, *next; unsigned long flags;

if (WARN_ON(index >= event->index_nr)) return;

spin_lock_irqsave(&event->list_lock, flags); - list_for_each_entry(handler, &event->list[index], head) { + list_for_each_entry_safe(handler, next, &event->list[index], head) { if (!test_bit(NVKM_EVENT_ENABLE, &handler->flags)) continue; if (!(handler->types & types)) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 2cf5b0572f66..6e71d3dc2d87 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -959,7 +959,7 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr, }

mutex_lock_nested(&chan->cli->mutex, SINGLE_DEPTH_NESTING); - ret = nouveau_fence_sync(bo->sync_obj, chan); + ret = nouveau_fence_sync(nouveau_bo(bo), chan); if (ret == 0) { ret = drm->ttm.move(chan, bo, &bo->mem, new_mem); if (ret == 0) { @@ -1432,10 +1432,12 @@ nouveau_bo_fence_unref(void **sync_obj) void nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence) { - lockdep_assert_held(&nvbo->bo.resv->lock.base); + struct reservation_object *resv = nvbo->bo.resv;

nouveau_bo_fence_unref(&nvbo->bo.sync_obj); nvbo->bo.sync_obj = nouveau_fence_ref(fence); + + reservation_object_add_excl_fence(resv, &fence->base); }

static void * diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index 7928f8f07334..2c4798750b20 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -660,7 +660,7 @@ nouveau_page_flip_emit(struct nouveau_channel *chan, spin_unlock_irqrestore(&dev->event_lock, flags);

/* Synchronize with the old framebuffer */ - ret = nouveau_fence_sync(old_bo->bo.sync_obj, chan); + ret = nouveau_fence_sync(old_bo, chan); if (ret) goto fail;

@@ -721,7 +721,7 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb, goto fail_unpin;

/* synchronise rendering channel with the kernel's channel */ - ret = nouveau_fence_sync(new_bo->bo.sync_obj, chan); + ret = nouveau_fence_sync(new_bo, chan); if (ret) { ttm_bo_unreserve(&new_bo->bo); goto fail_unpin; diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c index ab5ea3b0d666..d24f8ce4341a 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.c +++ b/drivers/gpu/drm/nouveau/nouveau_fence.c @@ -32,91 +32,139 @@ #include "nouveau_drm.h" #include "nouveau_dma.h" #include "nouveau_fence.h" +#include <trace/events/fence.h>

#include <engine/fifo.h>

-struct fence_work { - struct work_struct base; - struct list_head head; - void (*func)(void *); - void *data; -}; +static const struct fence_ops nouveau_fence_ops_uevent; +static const struct fence_ops nouveau_fence_ops_legacy;

static void nouveau_fence_signal(struct nouveau_fence *fence) { - struct fence_work *work, *temp; + fence_signal_locked(&fence->base); + list_del(&fence->head); + + if (fence->base.ops == &nouveau_fence_ops_uevent && + fence->event.head.next) { + struct nouveau_event *event;

- list_for_each_entry_safe(work, temp, &fence->work, head) { - schedule_work(&work->base); - list_del(&work->head); + list_del(&fence->event.head); + fence->event.head.next = NULL; + + event = container_of(fence->base.lock, typeof(*event), list_lock); + nouveau_event_put(&fence->event); }

- fence->channel = NULL; - list_del(&fence->head); + fence_put(&fence->base); +} + +static struct nouveau_fence * +nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) { + struct nouveau_fence_priv *priv = (void*)drm->fence; + struct nouveau_fence *f = container_of(fence, + struct nouveau_fence, + base); + + if (fence->ops != &nouveau_fence_ops_legacy && + fence->ops != &nouveau_fence_ops_uevent) + return NULL; + + if (fence->context < priv->context_base || + fence->context >= priv->context_base + priv->contexts) + return NULL; + + return f; }

void nouveau_fence_context_del(struct nouveau_fence_chan *fctx) { struct nouveau_fence *fence, *fnext; - spin_lock(&fctx->lock); - list_for_each_entry_safe(fence, fnext, &fctx->pending, head) { + + spin_lock_irq(fctx->lock); + list_for_each_entry_safe(fence, fnext, &fctx->pending, head) nouveau_fence_signal(fence); - } - spin_unlock(&fctx->lock); + spin_unlock_irq(fctx->lock); }

void -nouveau_fence_context_new(struct nouveau_fence_chan *fctx) +nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx) { + struct nouveau_fifo *pfifo = nouveau_fifo(chan->drm->device); + struct nouveau_fifo_chan *fifo = (void*)chan->object; + + fctx->lock = &pfifo->uevent->list_lock; INIT_LIST_HEAD(&fctx->flip); INIT_LIST_HEAD(&fctx->pending); - spin_lock_init(&fctx->lock); + + snprintf(fctx->name, sizeof(fctx->name) - 1, "nouveau channel %i", fifo->chid); }

+struct nouveau_fence_work { + struct work_struct work; + struct fence_cb cb; + void (*func)(void *); + void *data; +}; + static void nouveau_fence_work_handler(struct work_struct *kwork) { - struct fence_work *work = container_of(kwork, typeof(*work), base); + struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work); work->func(work->data); kfree(work); }

+static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb) +{ + struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb); + + schedule_work(&work->work); +} + +/* + * In an ideal world, read would not assume the channel context is still alive. + * This function may be called from another device, running into free memory as a + * result. The drm node should still be there, so we can derive the index from + * the fence context. + */ +static bool nouveau_fence_is_signaled(struct fence *f) +{ + struct nouveau_fence *fence = container_of(f, struct nouveau_fence, base); + struct nouveau_channel *chan = fence->channel; + struct nouveau_fence_chan *fctx = chan->fence; + + return (int)(fctx->read(chan) - fence->base.seqno) >= 0; +} + void nouveau_fence_work(struct nouveau_fence *fence, void (*func)(void *), void *data) { - struct nouveau_channel *chan = fence->channel; - struct nouveau_fence_chan *fctx; - struct fence_work *work = NULL; + struct nouveau_fence_work *work;

- if (nouveau_fence_done(fence)) { - func(data); - return; - } + if (fence_is_signaled(&fence->base)) + goto err;

- fctx = chan->fence; work = kmalloc(sizeof(*work), GFP_KERNEL); if (!work) { WARN_ON(nouveau_fence_wait(fence, false, false)); - func(data); - return; + goto err; }

- spin_lock(&fctx->lock); - if (!fence->channel) { - spin_unlock(&fctx->lock); - kfree(work); - func(data); - return; - } - - INIT_WORK(&work->base, nouveau_fence_work_handler); + INIT_WORK(&work->work, nouveau_fence_work_handler); work->func = func; work->data = data; - list_add(&work->head, &fence->work); - spin_unlock(&fctx->lock); + + if (fence_add_callback(&fence->base, &work->cb, nouveau_fence_work_cb) < 0) + goto err_free; + return; + +err_free: + kfree(work); +err: + func(data); }

static void @@ -125,33 +173,45 @@ nouveau_fence_update(struct nouveau_channel *chan) struct nouveau_fence_chan *fctx = chan->fence; struct nouveau_fence *fence, *fnext;

- spin_lock(&fctx->lock); + u32 seq = fctx->read(chan); + list_for_each_entry_safe(fence, fnext, &fctx->pending, head) { - if (fctx->read(chan) < fence->sequence) + if ((int)(seq - fence->base.seqno) < 0) break;

nouveau_fence_signal(fence); - nouveau_fence_unref(&fence); } - spin_unlock(&fctx->lock); }

int nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan) { struct nouveau_fence_chan *fctx = chan->fence; + struct nouveau_fifo *pfifo = nouveau_fifo(chan->drm->device); + struct nouveau_fifo_chan *fifo = (void*)chan->object; + struct nouveau_fence_priv *priv = (void*)chan->drm->fence; int ret;

fence->channel = chan; fence->timeout = jiffies + (15 * HZ); - fence->sequence = ++fctx->sequence;

+ if (priv->uevent) + fence_init(&fence->base, &nouveau_fence_ops_uevent, + &pfifo->uevent->list_lock, + priv->context_base + fifo->chid, ++fctx->sequence); + else + fence_init(&fence->base, &nouveau_fence_ops_legacy, + &pfifo->uevent->list_lock, + priv->context_base + fifo->chid, ++fctx->sequence); + + trace_fence_emit(&fence->base); ret = fctx->emit(fence); if (!ret) { - kref_get(&fence->kref); - spin_lock(&fctx->lock); + fence_get(&fence->base); + spin_lock_irq(fctx->lock); + nouveau_fence_update(chan); list_add_tail(&fence->head, &fctx->pending); - spin_unlock(&fctx->lock); + spin_unlock_irq(fctx->lock); }

return ret; @@ -160,104 +220,71 @@ nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan) bool nouveau_fence_done(struct nouveau_fence *fence) { - if (fence->channel) + if (fence->base.ops == &nouveau_fence_ops_legacy || + fence->base.ops == &nouveau_fence_ops_uevent) { + struct nouveau_fence_chan *fctx; + unsigned long flags; + + if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags)) + return true; + + fctx = fence->channel->fence; + spin_lock_irqsave(fctx->lock, flags); nouveau_fence_update(fence->channel); - return !fence->channel; + spin_unlock_irqrestore(fctx->lock, flags); + } + return fence_is_signaled(&fence->base); }

-static int -nouveau_fence_wait_uevent_handler(void *data, u32 type, int index) +static long +nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait) { - struct nouveau_fence_priv *priv = data; - wake_up_all(&priv->waiting); - return NVKM_EVENT_KEEP; -} + struct nouveau_fence *fence = container_of(f, typeof(*fence), base); + unsigned long sleep_time = NSEC_PER_MSEC / 1000; + unsigned long t = jiffies, timeout = t + wait;

-static int -nouveau_fence_wait_uevent(struct nouveau_fence *fence, bool intr) + while (!nouveau_fence_done(fence)) { + ktime_t kt;

-{ - struct nouveau_channel *chan = fence->channel; - struct nouveau_fifo *pfifo = nouveau_fifo(chan->drm->device); - struct nouveau_fence_priv *priv = chan->drm->fence; - struct nouveau_eventh *handler; - int ret = 0; + t = jiffies;

- ret = nouveau_event_new(pfifo->uevent, 1, 0, - nouveau_fence_wait_uevent_handler, - priv, &handler); - if (ret) - return ret; + if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) { + __set_current_state(TASK_RUNNING); + return 0; + }

- nouveau_event_get(handler); + __set_current_state(intr ? TASK_INTERRUPTIBLE : + TASK_UNINTERRUPTIBLE);

- if (fence->timeout) { - unsigned long timeout = fence->timeout - jiffies; - - if (time_before(jiffies, fence->timeout)) { - if (intr) { - ret = wait_event_interruptible_timeout( - priv->waiting, - nouveau_fence_done(fence), - timeout); - } else { - ret = wait_event_timeout(priv->waiting, - nouveau_fence_done(fence), - timeout); - } - } + kt = ktime_set(0, sleep_time); + schedule_hrtimeout(&kt, HRTIMER_MODE_REL); + sleep_time *= 2; + if (sleep_time > NSEC_PER_MSEC) + sleep_time = NSEC_PER_MSEC;

- if (ret >= 0) { - fence->timeout = jiffies + ret; - if (time_after_eq(jiffies, fence->timeout)) - ret = -EBUSY; - } - } else { - if (intr) { - ret = wait_event_interruptible(priv->waiting, - nouveau_fence_done(fence)); - } else { - wait_event(priv->waiting, nouveau_fence_done(fence)); - } + if (intr && signal_pending(current)) + return -ERESTARTSYS; }

- nouveau_event_ref(NULL, &handler); - if (unlikely(ret < 0)) - return ret; + __set_current_state(TASK_RUNNING);

- return 0; + return timeout - t; }

-int -nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr) +static int +nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr) { - struct nouveau_channel *chan = fence->channel; - struct nouveau_fence_priv *priv = chan ? chan->drm->fence : NULL; - unsigned long sleep_time = NSEC_PER_MSEC / 1000; - ktime_t t; int ret = 0;

- while (priv && priv->uevent && lazy && !nouveau_fence_done(fence)) { - ret = nouveau_fence_wait_uevent(fence, intr); - if (ret < 0) - return ret; - } - while (!nouveau_fence_done(fence)) { - if (fence->timeout && time_after_eq(jiffies, fence->timeout)) { + if (time_after_eq(jiffies, fence->timeout)) { ret = -EBUSY; break; }

- __set_current_state(intr ? TASK_INTERRUPTIBLE : - TASK_UNINTERRUPTIBLE); - if (lazy) { - t = ktime_set(0, sleep_time); - schedule_hrtimeout(&t, HRTIMER_MODE_REL); - sleep_time *= 2; - if (sleep_time > NSEC_PER_MSEC) - sleep_time = NSEC_PER_MSEC; - } + __set_current_state(intr ? + TASK_INTERRUPTIBLE : + TASK_UNINTERRUPTIBLE);

if (intr && signal_pending(current)) { ret = -ERESTARTSYS; @@ -270,36 +297,79 @@ nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr) }

int -nouveau_fence_sync(struct nouveau_fence *fence, struct nouveau_channel *chan) +nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr) +{ + long ret; + + if (!lazy) + return nouveau_fence_wait_busy(fence, intr); + + ret = fence_wait_timeout(&fence->base, intr, 15 * HZ); + if (ret < 0) + return ret; + else if (!ret) + return -EBUSY; + else + return 0; +} + +int +nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan) { struct nouveau_fence_chan *fctx = chan->fence; - struct nouveau_channel *prev; - int ret = 0; + struct fence *fence = NULL; + struct reservation_object *resv = nvbo->bo.resv; + struct reservation_object_list *fobj; + int ret = 0, i; + + fence = nvbo->bo.sync_obj; + if (fence && fence_is_signaled(fence)) { + nouveau_fence_unref((struct nouveau_fence **) + &nvbo->bo.sync_obj); + fence = NULL; + } + + if (fence) { + struct nouveau_fence *f = container_of(fence, + struct nouveau_fence, + base); + struct nouveau_channel *prev = f->channel;

- prev = fence ? fence->channel : NULL; - if (prev) { - if (unlikely(prev != chan && !nouveau_fence_done(fence))) { - ret = fctx->sync(fence, prev, chan); + if (prev != chan) { + ret = fctx->sync(f, prev, chan); if (unlikely(ret)) - ret = nouveau_fence_wait(fence, true, false); + ret = nouveau_fence_wait(f, true, true); } }

- return ret; -} + if (ret) + return ret;

-static void -nouveau_fence_del(struct kref *kref) -{ - struct nouveau_fence *fence = container_of(kref, typeof(*fence), kref); - kfree(fence); + fence = reservation_object_get_excl(resv); + if (fence && !nouveau_local_fence(fence, chan->drm)) + ret = fence_wait(fence, true); + + fobj = reservation_object_get_list(resv); + if (!fobj || ret) + return ret; + + for (i = 0; i < fobj->shared_count && !ret; ++i) { + fence = rcu_dereference_protected(fobj->shared[i], + reservation_object_held(resv)); + + /* should always be true, for now */ + if (!nouveau_local_fence(fence, chan->drm)) + ret = fence_wait(fence, true); + } + + return ret; }

void nouveau_fence_unref(struct nouveau_fence **pfence) { if (*pfence) - kref_put(&(*pfence)->kref, nouveau_fence_del); + fence_put(&(*pfence)->base); *pfence = NULL; }

@@ -307,7 +377,7 @@ struct nouveau_fence * nouveau_fence_ref(struct nouveau_fence *fence) { if (fence) - kref_get(&fence->kref); + fence_get(&fence->base); return fence; }

@@ -325,9 +395,7 @@ nouveau_fence_new(struct nouveau_channel *chan, bool sysmem, if (!fence) return -ENOMEM;

- INIT_LIST_HEAD(&fence->work); fence->sysmem = sysmem; - kref_init(&fence->kref);

ret = nouveau_fence_emit(fence, chan); if (ret) @@ -336,3 +404,86 @@ nouveau_fence_new(struct nouveau_channel *chan, bool sysmem, *pfence = fence; return ret; } + + +static bool nouveau_fence_no_signaling(struct fence *f) +{ + /* + * This needs uevents to work correctly, but fence_add_callback relies on + * being able to enable signaling. It will still get signaled eventually, + * just not right away. + */ + if (nouveau_fence_is_signaled(f)) + return false; + + return true; +} + +static const char *nouveau_fence_get_get_driver_name(struct fence *fence) +{ + return "nouveau"; +} + +static const char *nouveau_fence_get_timeline_name(struct fence *f) +{ + struct nouveau_fence *fence = + container_of(f, struct nouveau_fence, base); + struct nouveau_fence_chan *fctx = fence->channel->fence; + + return fctx ? fctx->name : "dead channel"; +} + +static const struct fence_ops nouveau_fence_ops_legacy = { + .get_driver_name = nouveau_fence_get_get_driver_name, + .get_timeline_name = nouveau_fence_get_timeline_name, + .enable_signaling = nouveau_fence_no_signaling, + .signaled = nouveau_fence_is_signaled, + .wait = nouveau_fence_wait_legacy, + .release = NULL +}; + +static int +nouveau_fence_wait_uevent_handler(void *priv, u32 types, int index) +{ + struct nouveau_fence *fence = priv; + + if (nouveau_fence_is_signaled(&fence->base)) + nouveau_fence_signal(fence); + + /* + * NVKM_EVENT_DROP is never appropriate here, nouveau_fence_signal + * will unlink and free the event if needed. + */ + return NVKM_EVENT_KEEP; +} + +static bool nouveau_fence_enable_signaling(struct fence *f) +{ + struct nouveau_fence *fence = container_of(f, struct nouveau_fence, base); + struct nouveau_event *event = container_of(f->lock, struct nouveau_event, list_lock); + struct nouveau_eventh *handler = &fence->event; + + handler->event = event; + handler->func = nouveau_fence_wait_uevent_handler; + handler->priv = fence; + handler->types = 1; + + nouveau_event_get(handler); + if (nouveau_fence_is_signaled(f)) { + nouveau_event_put(handler); + return false; + } + + list_add_tail(&handler->head, &event->list[0]); + + return true; +} + +static const struct fence_ops nouveau_fence_ops_uevent = { + .get_driver_name = nouveau_fence_get_get_driver_name, + .get_timeline_name = nouveau_fence_get_timeline_name, + .enable_signaling = nouveau_fence_enable_signaling, + .signaled = nouveau_fence_is_signaled, + .wait = fence_default_wait, + .release = NULL +}; diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h index c57bb61da58c..1989ec22e66e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.h +++ b/drivers/gpu/drm/nouveau/nouveau_fence.h @@ -1,18 +1,21 @@ #ifndef __NOUVEAU_FENCE_H__ #define __NOUVEAU_FENCE_H__

+#include <linux/fence.h> + struct nouveau_drm; +struct nouveau_bo;

struct nouveau_fence { + struct fence base; + struct list_head head; - struct list_head work; - struct kref kref; + struct nouveau_eventh event;

bool sysmem;

struct nouveau_channel *channel; unsigned long timeout; - u32 sequence; };

int nouveau_fence_new(struct nouveau_channel *, bool sysmem, @@ -25,7 +28,7 @@ int nouveau_fence_emit(struct nouveau_fence *, struct nouveau_channel *); bool nouveau_fence_done(struct nouveau_fence *); void nouveau_fence_work(struct nouveau_fence *, void (*)(void *), void *); int nouveau_fence_wait(struct nouveau_fence *, bool lazy, bool intr); -int nouveau_fence_sync(struct nouveau_fence *, struct nouveau_channel *); +int nouveau_fence_sync(struct nouveau_bo *, struct nouveau_channel *);

struct nouveau_fence_chan { struct list_head pending; @@ -38,8 +41,10 @@ struct nouveau_fence_chan { int (*emit32)(struct nouveau_channel *, u64, u32); int (*sync32)(struct nouveau_channel *, u64, u32);

- spinlock_t lock; + spinlock_t *lock; u32 sequence; + u32 context; + char name[24]; };

struct nouveau_fence_priv { @@ -49,13 +54,14 @@ struct nouveau_fence_priv { int (*context_new)(struct nouveau_channel *); void (*context_del)(struct nouveau_channel *);

- wait_queue_head_t waiting; bool uevent; + + u32 contexts, context_base; };

#define nouveau_fence(drm) ((struct nouveau_fence_priv *)(drm)->fence)

-void nouveau_fence_context_new(struct nouveau_fence_chan *); +void nouveau_fence_context_new(struct nouveau_channel *, struct nouveau_fence_chan *); void nouveau_fence_context_del(struct nouveau_fence_chan *);

int nv04_fence_create(struct nouveau_drm *); diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 474f67238cef..2bd62f4ecf6a 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -428,18 +428,6 @@ retry: }

static int -validate_sync(struct nouveau_channel *chan, struct nouveau_bo *nvbo) -{ - struct nouveau_fence *fence = nvbo->bo.sync_obj; - int ret = 0; - - if (fence) - ret = nouveau_fence_sync(fence, chan); - - return ret; -} - -static int validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo, uint64_t user_pbbo_ptr) @@ -468,9 +456,10 @@ validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, return ret; }

- ret = validate_sync(chan, nvbo); + ret = nouveau_fence_sync(nvbo, chan); if (unlikely(ret)) { - NV_ERROR(cli, "fail post-validate sync\n"); + if (ret != -ERESTARTSYS) + NV_ERROR(cli, "fail post-validate sync\n"); return ret; }

diff --git a/drivers/gpu/drm/nouveau/nv04_fence.c b/drivers/gpu/drm/nouveau/nv04_fence.c index 94eadd1dd10a..997c54122ed9 100644 --- a/drivers/gpu/drm/nouveau/nv04_fence.c +++ b/drivers/gpu/drm/nouveau/nv04_fence.c @@ -43,7 +43,7 @@ nv04_fence_emit(struct nouveau_fence *fence) int ret = RING_SPACE(chan, 2); if (ret == 0) { BEGIN_NV04(chan, NvSubSw, 0x0150, 1); - OUT_RING (chan, fence->sequence); + OUT_RING (chan, fence->base.seqno); FIRE_RING (chan); } return ret; @@ -77,7 +77,7 @@ nv04_fence_context_new(struct nouveau_channel *chan) { struct nv04_fence_chan *fctx = kzalloc(sizeof(*fctx), GFP_KERNEL); if (fctx) { - nouveau_fence_context_new(&fctx->base); + nouveau_fence_context_new(chan, &fctx->base); fctx->base.emit = nv04_fence_emit; fctx->base.sync = nv04_fence_sync; fctx->base.read = nv04_fence_read; diff --git a/drivers/gpu/drm/nouveau/nv10_fence.c b/drivers/gpu/drm/nouveau/nv10_fence.c index 06f434f03fba..e8f73f7f31ef 100644 --- a/drivers/gpu/drm/nouveau/nv10_fence.c +++ b/drivers/gpu/drm/nouveau/nv10_fence.c @@ -36,7 +36,7 @@ nv10_fence_emit(struct nouveau_fence *fence) int ret = RING_SPACE(chan, 2); if (ret == 0) { BEGIN_NV04(chan, 0, NV10_SUBCHAN_REF_CNT, 1); - OUT_RING (chan, fence->sequence); + OUT_RING (chan, fence->base.seqno); FIRE_RING (chan); } return ret; @@ -74,7 +74,7 @@ nv10_fence_context_new(struct nouveau_channel *chan) if (!fctx) return -ENOMEM;

- nouveau_fence_context_new(&fctx->base); + nouveau_fence_context_new(chan, &fctx->base); fctx->base.emit = nv10_fence_emit; fctx->base.read = nv10_fence_read; fctx->base.sync = nv10_fence_sync; diff --git a/drivers/gpu/drm/nouveau/nv17_fence.c b/drivers/gpu/drm/nouveau/nv17_fence.c index 22aa9963ea6f..e404bab31e9d 100644 --- a/drivers/gpu/drm/nouveau/nv17_fence.c +++ b/drivers/gpu/drm/nouveau/nv17_fence.c @@ -83,7 +83,7 @@ nv17_fence_context_new(struct nouveau_channel *chan) if (!fctx) return -ENOMEM;

- nouveau_fence_context_new(&fctx->base); + nouveau_fence_context_new(chan, &fctx->base); fctx->base.emit = nv10_fence_emit; fctx->base.read = nv10_fence_read; fctx->base.sync = nv17_fence_sync; diff --git a/drivers/gpu/drm/nouveau/nv50_fence.c b/drivers/gpu/drm/nouveau/nv50_fence.c index 0ee363840035..19f6fccb84a1 100644 --- a/drivers/gpu/drm/nouveau/nv50_fence.c +++ b/drivers/gpu/drm/nouveau/nv50_fence.c @@ -47,7 +47,7 @@ nv50_fence_context_new(struct nouveau_channel *chan) if (!fctx) return -ENOMEM;

- nouveau_fence_context_new(&fctx->base); + nouveau_fence_context_new(chan, &fctx->base); fctx->base.emit = nv10_fence_emit; fctx->base.read = nv10_fence_read; fctx->base.sync = nv17_fence_sync; diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c index 9fd475c89820..8a06727b23d1 100644 --- a/drivers/gpu/drm/nouveau/nv84_fence.c +++ b/drivers/gpu/drm/nouveau/nv84_fence.c @@ -89,7 +89,7 @@ nv84_fence_emit(struct nouveau_fence *fence) else addr += fctx->vma.offset;

- return fctx->base.emit32(chan, addr, fence->sequence); + return fctx->base.emit32(chan, addr, fence->base.seqno); }

static int @@ -105,7 +105,7 @@ nv84_fence_sync(struct nouveau_fence *fence, else addr += fctx->vma.offset;

- return fctx->base.sync32(chan, addr, fence->sequence); + return fctx->base.sync32(chan, addr, fence->base.seqno); }

static u32 @@ -149,12 +149,14 @@ nv84_fence_context_new(struct nouveau_channel *chan) if (!fctx) return -ENOMEM;

- nouveau_fence_context_new(&fctx->base); + nouveau_fence_context_new(chan, &fctx->base); fctx->base.emit = nv84_fence_emit; fctx->base.sync = nv84_fence_sync; fctx->base.read = nv84_fence_read; fctx->base.emit32 = nv84_fence_emit32; fctx->base.sync32 = nv84_fence_sync32; + fctx->base.sequence = nv84_fence_read(chan); + fctx->base.context = priv->base.context_base + fifo->chid;

ret = nouveau_bo_vma_add(priv->bo, client->vm, &fctx->vma); if (ret == 0) { @@ -239,7 +241,8 @@ nv84_fence_create(struct nouveau_drm *drm) priv->base.context_new = nv84_fence_context_new; priv->base.context_del = nv84_fence_context_del;

- init_waitqueue_head(&priv->base.waiting); + priv->base.contexts = pfifo->max + 1; + priv->base.context_base = fence_context_alloc(priv->base.contexts); priv->base.uevent = true;

ret = nouveau_bo_new(drm->dev, 16 * (pfifo->max + 1), 0,

Changes since v1: - Kill the sw interrupt dance, add and use radeon_irq_kms_sw_irq_get_delayed instead. - Change custom wait function, lockdep complained about it. Holding exclusive_lock in the wait function might cause deadlocks. Instead do all the processing in .enable_signaling, and wait on the global fence_queue to pick up gpu resets. - Process all fences in radeon_gpu_reset after reset to close a race with the trylock in enable_signaling.

Signed-off-by: Maarten Lankhorst maarten.lankhorst@canonical.com --- drivers/gpu/drm/radeon/radeon.h | 18 +- drivers/gpu/drm/radeon/radeon_device.c | 22 ++- drivers/gpu/drm/radeon/radeon_fence.c | 263 +++++++++++++++++++++++++++---- drivers/gpu/drm/radeon/radeon_irq_kms.c | 43 +++++ 4 files changed, 293 insertions(+), 53 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index b01d88fc10cb..599bb28e4104 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -64,6 +64,7 @@ #include <linux/wait.h> #include <linux/list.h> #include <linux/kref.h> +#include <linux/fence.h>

#include <ttm/ttm_bo_api.h> #include <ttm/ttm_bo_driver.h> @@ -116,9 +117,6 @@ extern int radeon_deep_color; #define RADEONFB_CONN_LIMIT 4 #define RADEON_BIOS_NUM_SCRATCH 8

-/* fence seq are set to this number when signaled */ -#define RADEON_FENCE_SIGNALED_SEQ 0LL - /* internal ring indices */ /* r1xx+ has gfx CP ring */ #define RADEON_RING_TYPE_GFX_INDEX 0 @@ -352,12 +350,15 @@ struct radeon_fence_driver { };

struct radeon_fence { + struct fence base; + struct radeon_device *rdev; - struct kref kref; /* protected by radeon_fence.lock */ uint64_t seq; /* RB, DMA, etc. */ unsigned ring; + + wait_queue_t fence_wake; };

int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring); @@ -769,6 +770,7 @@ struct radeon_irq { int radeon_irq_kms_init(struct radeon_device *rdev); void radeon_irq_kms_fini(struct radeon_device *rdev); void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring); +void radeon_irq_kms_sw_irq_get_delayed(struct radeon_device *rdev, int ring); void radeon_irq_kms_sw_irq_put(struct radeon_device *rdev, int ring); void radeon_irq_kms_pflip_irq_get(struct radeon_device *rdev, int crtc); void radeon_irq_kms_pflip_irq_put(struct radeon_device *rdev, int crtc); @@ -2276,6 +2278,7 @@ struct radeon_device { struct radeon_mman mman; struct radeon_fence_driver fence_drv[RADEON_NUM_RINGS]; wait_queue_head_t fence_queue; + unsigned fence_context; struct mutex ring_lock; struct radeon_ring ring[RADEON_NUM_RINGS]; bool ib_pool_ready; @@ -2314,6 +2317,8 @@ struct radeon_device { struct work_struct hotplug_work; struct work_struct audio_work; struct work_struct reset_work; + struct work_struct delayed_irq_work; + int num_crtc; /* number of crtcs */ struct mutex dc_hw_i2c_mutex; /* display controller hw i2c mutex */ bool has_uvd; @@ -2366,11 +2371,6 @@ u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index); void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v);

/* - * Cast helper - */ -#define to_radeon_fence(p) ((struct radeon_fence *)(p)) - -/* * Registers read & write functions. */ #define RREG8(reg) readb((rdev->rmmio) + (reg)) diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index 21efd32d07ee..dbf3a8bc91d2 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -1213,6 +1213,7 @@ int radeon_device_init(struct radeon_device *rdev, for (i = 0; i < RADEON_NUM_RINGS; i++) { rdev->ring[i].idx = i; } + rdev->fence_context = fence_context_alloc(RADEON_NUM_RINGS);

DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X).\n", radeon_family_name[rdev->family], pdev->vendor, pdev->device, @@ -1622,15 +1623,13 @@ int radeon_gpu_reset(struct radeon_device *rdev)

bool saved = false;

- int i, r; + int i, r = 0; int resched;

down_write(&rdev->exclusive_lock);

- if (!rdev->needs_reset) { - up_write(&rdev->exclusive_lock); - return 0; - } + if (!rdev->needs_reset) + goto out;

rdev->needs_reset = false;

@@ -1697,7 +1696,18 @@ retry: dev_info(rdev->dev, "GPU reset failed\n"); }

- up_write(&rdev->exclusive_lock); +out: + /* + * force all waiters to recheck, some may have been + * added while the exclusive_lock was unavailable + */ + downgrade_write(&rdev->exclusive_lock); + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + if (rdev->ring[i].ready) + radeon_fence_process(rdev, i); + } + wake_up_all(&rdev->fence_queue); + up_read(&rdev->exclusive_lock); return r; }

diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index f79e0b34582b..8ca56ca70503 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -39,6 +39,15 @@ #include "radeon.h" #include "radeon_trace.h"

+static const struct fence_ops radeon_fence_ops; + +#define to_radeon_fence(p) \ + ({ \ + struct radeon_fence *__f; \ + __f = container_of((p), struct radeon_fence, base); \ + __f->base.ops == &radeon_fence_ops ? __f : NULL; \ + }) + /* * Fences * Fences mark an event in the GPUs pipeline and are used @@ -111,20 +120,82 @@ int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence **fence, int ring) { + u64 seq = ++rdev->fence_drv[ring].sync_seq[ring]; + /* we are protected by the ring emission mutex */ *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL); if ((*fence) == NULL) { return -ENOMEM; } - kref_init(&((*fence)->kref)); - (*fence)->rdev = rdev; - (*fence)->seq = ++rdev->fence_drv[ring].sync_seq[ring]; (*fence)->ring = ring; + fence_init(&(*fence)->base, &radeon_fence_ops, + &rdev->fence_queue.lock, rdev->fence_context + ring, seq); + (*fence)->rdev = rdev; + (*fence)->seq = seq; radeon_fence_ring_emit(rdev, ring, *fence); trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); return 0; }

+/** + * __radeon_fence_read_seq - Returns the current fence value without updating + * + * @rdev: radeon_device pointer + * @ring: ring index to return the seqno of + */ +static uint64_t __radeon_fence_read_seq(struct radeon_device *rdev, int ring) +{ + uint64_t last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq); + uint64_t last_emitted = rdev->fence_drv[ring].sync_seq[ring]; + uint64_t seq = radeon_fence_read(rdev, ring); + + seq = radeon_fence_read(rdev, ring); + seq |= last_seq & 0xffffffff00000000LL; + if (seq < last_seq) { + seq &= 0xffffffff; + seq |= last_emitted & 0xffffffff00000000LL; + } + return seq; +} + +/** + * radeon_fence_check_signaled - callback from fence_queue + * + * this function is called with fence_queue lock held, which is also used + * for the fence locking itself, so unlocked variants are used for + * fence_signal, and remove_wait_queue. + */ +static int radeon_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key) +{ + struct radeon_fence *fence; + u64 seq; + + fence = container_of(wait, struct radeon_fence, fence_wake); + + /* + * We cannot use radeon_fence_process here because we're already + * in the waitqueue, in a call from wake_up_all. + * + * Because the current ring might not have been updated, call + * __radeon_fence_read_seq here to check without updating. + */ + seq = __radeon_fence_read_seq(fence->rdev, fence->ring); + if (seq >= fence->seq) { + int ret = fence_signal_locked(&fence->base); + + if (!ret) + FENCE_TRACE(&fence->base, "signaled from irq context\n"); + else + FENCE_TRACE(&fence->base, "was already signaled\n"); + + radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring); + __remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake); + fence_put(&fence->base); + } else + FENCE_TRACE(&fence->base, "pending\n"); + return 0; +} + static bool __radeon_fence_process(struct radeon_device *rdev, int ring) { uint64_t seq, last_seq, last_emitted; @@ -230,21 +301,6 @@ void radeon_fence_process(struct radeon_device *rdev, int ring) }

/** - * radeon_fence_destroy - destroy a fence - * - * @kref: fence kref - * - * Frees the fence object (all asics). - */ -static void radeon_fence_destroy(struct kref *kref) -{ - struct radeon_fence *fence; - - fence = container_of(kref, struct radeon_fence, kref); - kfree(fence); -} - -/** * radeon_fence_seq_signaled - check if a fence sequence number has signaled * * @rdev: radeon device pointer @@ -272,6 +328,73 @@ static bool radeon_fence_seq_signaled(struct radeon_device *rdev, return false; }

+static bool __radeon_fence_signaled(struct fence *f) +{ + struct radeon_fence *fence = to_radeon_fence(f); + struct radeon_device *rdev = fence->rdev; + unsigned ring = fence->ring; + u64 seq = fence->seq; + + if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { + return true; + } + + if (down_read_trylock(&rdev->exclusive_lock)) { + radeon_fence_process(rdev, ring); + up_read(&rdev->exclusive_lock); + + if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { + return true; + } + } + return false; +} + +/** + * radeon_fence_enable_signaling - enable signalling on fence + * @fence: fence + * + * This function is called with fence_queue lock held, and adds a callback + * to fence_queue that checks if this fence is signaled, and if so it + * signals the fence and removes itself. + */ +static bool radeon_fence_enable_signaling(struct fence *f) +{ + struct radeon_fence *fence = to_radeon_fence(f); + struct radeon_device *rdev = fence->rdev; + + if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq || + !rdev->ddev->irq_enabled) + return false; + + if (down_read_trylock(&rdev->exclusive_lock)) { + radeon_irq_kms_sw_irq_get(rdev, fence->ring); + + if (__radeon_fence_process(rdev, fence->ring)) + wake_up_all_locked(&rdev->fence_queue); + + /* did fence get signaled after we enabled the sw irq? */ + if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) { + radeon_irq_kms_sw_irq_put(rdev, fence->ring); + up_read(&rdev->exclusive_lock); + return false; + } + + up_read(&rdev->exclusive_lock); + } else + /* we're probably in a lockup, lets not fiddle too much */ + radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring); + + fence->fence_wake.flags = 0; + fence->fence_wake.private = NULL; + fence->fence_wake.func = radeon_fence_check_signaled; + __add_wait_queue(&rdev->fence_queue, &fence->fence_wake); + fence_get(f); + + FENCE_TRACE(&fence->base, "armed on ring %i!\n", fence->ring); + return true; +} + /** * radeon_fence_signaled - check if a fence has signaled * @@ -285,11 +408,13 @@ bool radeon_fence_signaled(struct radeon_fence *fence) if (!fence) { return true; } - if (fence->seq == RADEON_FENCE_SIGNALED_SEQ) { - return true; - } + if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { - fence->seq = RADEON_FENCE_SIGNALED_SEQ; + int ret; + + ret = fence_signal(&fence->base); + if (!ret) + FENCE_TRACE(&fence->base, "signaled from radeon_fence_signaled\n"); return true; } return false; @@ -400,21 +525,18 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr) uint64_t seq[RADEON_NUM_RINGS] = {}; long r;

- if (fence == NULL) { - WARN(1, "Querying an invalid fence : %p !\n", fence); - return -EINVAL; - } - - seq[fence->ring] = fence->seq; - if (seq[fence->ring] == RADEON_FENCE_SIGNALED_SEQ) + if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags)) return 0;

+ seq[fence->ring] = fence->seq; r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, MAX_SCHEDULE_TIMEOUT); if (r < 0) { return r; }

- fence->seq = RADEON_FENCE_SIGNALED_SEQ; + r = fence_signal(&fence->base); + if (!r) + FENCE_TRACE(&fence->base, "signaled from fence_wait\n"); return 0; }

@@ -446,12 +568,13 @@ int radeon_fence_wait_any(struct radeon_device *rdev, continue; }

+ if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fences[i]->base.flags)) { + /* already signaled */ + return 0; + } + seq[i] = fences[i]->seq; ++num_rings; - - /* test if something was allready signaled */ - if (seq[i] == RADEON_FENCE_SIGNALED_SEQ) - return 0; }

/* nothing to wait for ? */ @@ -532,7 +655,7 @@ int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) */ struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence) { - kref_get(&fence->kref); + fence_get(&fence->base); return fence; }

@@ -548,9 +671,8 @@ void radeon_fence_unref(struct radeon_fence **fence) struct radeon_fence *tmp = *fence;

*fence = NULL; - if (tmp) { - kref_put(&tmp->kref, radeon_fence_destroy); - } + if (tmp) + fence_put(&tmp->base); }

/** @@ -855,6 +977,7 @@ static int radeon_debugfs_gpu_reset(struct seq_file *m, void *data) down_read(&rdev->exclusive_lock); seq_printf(m, "%d\n", rdev->needs_reset); rdev->needs_reset = true; + wake_up_all(&rdev->fence_queue); up_read(&rdev->exclusive_lock);

return 0; @@ -874,3 +997,67 @@ int radeon_debugfs_fence_init(struct radeon_device *rdev) return 0; #endif } + +static const char *radeon_fence_get_driver_name(struct fence *fence) +{ + return "radeon"; +} + +static const char *radeon_fence_get_timeline_name(struct fence *f) +{ + struct radeon_fence *fence = to_radeon_fence(f); + switch (fence->ring) { + case RADEON_RING_TYPE_GFX_INDEX: return "radeon.gfx"; + case CAYMAN_RING_TYPE_CP1_INDEX: return "radeon.cp1"; + case CAYMAN_RING_TYPE_CP2_INDEX: return "radeon.cp2"; + case R600_RING_TYPE_DMA_INDEX: return "radeon.dma"; + case CAYMAN_RING_TYPE_DMA1_INDEX: return "radeon.dma1"; + case R600_RING_TYPE_UVD_INDEX: return "radeon.uvd"; + case TN_RING_TYPE_VCE1_INDEX: return "radeon.vce1"; + case TN_RING_TYPE_VCE2_INDEX: return "radeon.vce2"; + default: WARN_ON_ONCE(1); return "radeon.unk"; + } +} + +static signed long __radeon_fence_default_wait(struct fence *f, bool intr, + signed long timeout) +{ + struct radeon_fence *fence = to_radeon_fence(f); + struct radeon_device *rdev = fence->rdev; + bool signaled; + + fence_enable_sw_signaling(&fence->base); + + /* + * This function has to return -EDEADLK, but cannot hold + * exclusive_lock during the wait because some callers + * may already hold it. This means checking needs_reset without + * lock, and not fiddling with any gpu internals. + * + * The callback installed with fence_enable_sw_signaling will + * run before our wait_event_*timeout call, so we will see + * both the signaled fence and the changes to needs_reset. + */ + + if (intr) + timeout = wait_event_interruptible_timeout(rdev->fence_queue, + ((signaled = (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))) || rdev->needs_reset), + timeout); + else + timeout = wait_event_timeout(rdev->fence_queue, + ((signaled = (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))) || rdev->needs_reset), + timeout); + + if (timeout > 0 && !signaled) + return -EDEADLK; + return timeout; +} + +static const struct fence_ops radeon_fence_ops = { + .get_driver_name = radeon_fence_get_driver_name, + .get_timeline_name = radeon_fence_get_timeline_name, + .enable_signaling = radeon_fence_enable_signaling, + .signaled = __radeon_fence_signaled, + .wait = __radeon_fence_default_wait, + .release = NULL, +}; diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c index 16807afab362..41310cec61a6 100644 --- a/drivers/gpu/drm/radeon/radeon_irq_kms.c +++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c @@ -254,6 +254,25 @@ static bool radeon_msi_ok(struct radeon_device *rdev) return true; }

+static void radeon_delayed_irq_set(struct work_struct *work) +{ + struct radeon_device *rdev; + unsigned long irqflags; + + rdev = container_of(work, struct radeon_device, delayed_irq_work); + + down_read(&rdev->exclusive_lock); + + spin_lock_irqsave(&rdev->irq.lock, irqflags); + radeon_irq_set(rdev); + spin_unlock_irqrestore(&rdev->irq.lock, irqflags); + + /* force all waiters to recheck */ + wake_up_all(&rdev->fence_queue); + + up_read(&rdev->exclusive_lock); +} + /** * radeon_irq_kms_init - init driver interrupt info * @@ -285,6 +304,7 @@ int radeon_irq_kms_init(struct radeon_device *rdev) INIT_WORK(&rdev->hotplug_work, radeon_hotplug_work_func); INIT_WORK(&rdev->audio_work, r600_audio_update_hdmi); INIT_WORK(&rdev->reset_work, radeon_irq_reset_work_func); + INIT_WORK(&rdev->delayed_irq_work, radeon_delayed_irq_set);

rdev->irq.installed = true; r = drm_irq_install(rdev->ddev, rdev->ddev->pdev->irq); @@ -313,6 +333,7 @@ void radeon_irq_kms_fini(struct radeon_device *rdev) rdev->irq.installed = false; if (rdev->msi_enabled) pci_disable_msi(rdev->pdev); + cancel_work_sync(&rdev->delayed_irq_work); flush_work(&rdev->hotplug_work); } } @@ -342,6 +363,28 @@ void radeon_irq_kms_sw_irq_get(struct radeon_device *rdev, int ring) }

/** + * radeon_irq_kms_sw_irq_get_delayed - enable software interrupt + * + * @rdev: radeon device pointer + * @ring: ring whose interrupt you want to enable + * + * Enables the software interrupt for a specific ring (all asics). + * The software interrupt is generally used to signal a fence on + * a particular ring. + * + * This function can be used without exclusive_lock grabbed, + * and will schedule later work for irqs to be set when needed. + */ +void radeon_irq_kms_sw_irq_get_delayed(struct radeon_device *rdev, int ring) +{ + if (!rdev->ddev->irq_enabled) + return; + + if (atomic_inc_return(&rdev->irq.ring_int[ring]) == 1) + schedule_work(&rdev->delayed_irq_work); +} + +/** * radeon_irq_kms_sw_irq_put - disable software interrupt * * @rdev: radeon device pointer

Only one type was ever used. This is needed to simplify the fence support in the next commit.

Signed-off-by: Maarten Lankhorst maarten.lankhorst@canonical.com --- drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c | 5 +-- drivers/gpu/drm/vmwgfx/vmwgfx_drv.h | 1 - drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c | 14 ++------- drivers/gpu/drm/vmwgfx/vmwgfx_fence.c | 50 ++++++++++++------------------- drivers/gpu/drm/vmwgfx/vmwgfx_fence.h | 8 +---- 5 files changed, 26 insertions(+), 52 deletions(-)

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c index 4a36bb1dc525..f15718cc631d 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c @@ -792,15 +792,12 @@ static int vmw_sync_obj_flush(void *sync_obj)

static bool vmw_sync_obj_signaled(void *sync_obj) { - return vmw_fence_obj_signaled((struct vmw_fence_obj *) sync_obj, - DRM_VMW_FENCE_FLAG_EXEC); - + return vmw_fence_obj_signaled((struct vmw_fence_obj *) sync_obj); }

static int vmw_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible) { return vmw_fence_obj_wait((struct vmw_fence_obj *) sync_obj, - DRM_VMW_FENCE_FLAG_EXEC, lazy, interruptible, VMW_FENCE_WAIT_TIMEOUT); } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h index c1811750cc8d..0b1e4b0c919e 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h @@ -342,7 +342,6 @@ struct vmw_sw_context{ uint32_t *cmd_bounce; uint32_t cmd_bounce_size; struct list_head resource_list; - uint32_t fence_flags; struct ttm_buffer_object *cur_query_bo; struct list_head res_relocations; uint32_t *buf_start; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index b19b2b980cb4..214fcfd13cb2 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -350,8 +350,6 @@ static int vmw_bo_to_validate_list(struct vmw_sw_context *sw_context, vval_buf->validate_as_mob = validate_as_mob; }

- sw_context->fence_flags |= DRM_VMW_FENCE_FLAG_EXEC; - if (p_val_node) *p_val_node = val_node;

@@ -2337,13 +2335,9 @@ int vmw_execbuf_fence_commands(struct drm_file *file_priv,

if (p_handle != NULL) ret = vmw_user_fence_create(file_priv, dev_priv->fman, - sequence, - DRM_VMW_FENCE_FLAG_EXEC, - p_fence, p_handle); + sequence, p_fence, p_handle); else - ret = vmw_fence_create(dev_priv->fman, sequence, - DRM_VMW_FENCE_FLAG_EXEC, - p_fence); + ret = vmw_fence_create(dev_priv->fman, sequence, p_fence);

if (unlikely(ret != 0 && !synced)) { (void) vmw_fallback_wait(dev_priv, false, false, @@ -2416,8 +2410,7 @@ vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv, ttm_ref_object_base_unref(vmw_fp->tfile, fence_handle, TTM_REF_USAGE); DRM_ERROR("Fence copy error. Syncing.\n"); - (void) vmw_fence_obj_wait(fence, fence->signal_mask, - false, false, + (void) vmw_fence_obj_wait(fence, false, false, VMW_FENCE_WAIT_TIMEOUT); } } @@ -2469,7 +2462,6 @@ int vmw_execbuf_process(struct drm_file *file_priv, sw_context->fp = vmw_fpriv(file_priv); sw_context->cur_reloc = 0; sw_context->cur_val_buf = 0; - sw_context->fence_flags = 0; INIT_LIST_HEAD(&sw_context->resource_list); sw_context->cur_query_bo = dev_priv->pinned_bo; sw_context->last_query_ctx = NULL; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c index 436b013b4231..05b9eea8e875 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c @@ -207,9 +207,7 @@ void vmw_fence_manager_takedown(struct vmw_fence_manager *fman) }

static int vmw_fence_obj_init(struct vmw_fence_manager *fman, - struct vmw_fence_obj *fence, - u32 seqno, - uint32_t mask, + struct vmw_fence_obj *fence, u32 seqno, void (*destroy) (struct vmw_fence_obj *fence)) { unsigned long irq_flags; @@ -220,7 +218,6 @@ static int vmw_fence_obj_init(struct vmw_fence_manager *fman, INIT_LIST_HEAD(&fence->seq_passed_actions); fence->fman = fman; fence->signaled = 0; - fence->signal_mask = mask; kref_init(&fence->kref); fence->destroy = destroy; init_waitqueue_head(&fence->queue); @@ -356,7 +353,7 @@ static bool vmw_fence_goal_check_locked(struct vmw_fence_obj *fence) u32 goal_seqno; __le32 __iomem *fifo_mem;

- if (fence->signaled & DRM_VMW_FENCE_FLAG_EXEC) + if (fence->signaled) return false;

fifo_mem = fence->fman->dev_priv->mmio_virt; @@ -386,7 +383,7 @@ rerun: list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) { if (seqno - fence->seqno < VMW_FENCE_WRAP) { list_del_init(&fence->head); - fence->signaled |= DRM_VMW_FENCE_FLAG_EXEC; + fence->signaled = 1; INIT_LIST_HEAD(&action_list); list_splice_init(&fence->seq_passed_actions, &action_list); @@ -417,8 +414,7 @@ rerun: } }

-bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence, - uint32_t flags) +bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence) { struct vmw_fence_manager *fman = fence->fman; unsigned long irq_flags; @@ -428,28 +424,25 @@ bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence, signaled = fence->signaled; spin_unlock_irqrestore(&fman->lock, irq_flags);

- flags &= fence->signal_mask; - if ((signaled & flags) == flags) + if (signaled) return 1;

- if ((signaled & DRM_VMW_FENCE_FLAG_EXEC) == 0) - vmw_fences_update(fman); + vmw_fences_update(fman);

spin_lock_irqsave(&fman->lock, irq_flags); signaled = fence->signaled; spin_unlock_irqrestore(&fman->lock, irq_flags);

- return ((signaled & flags) == flags); + return signaled; }

-int vmw_fence_obj_wait(struct vmw_fence_obj *fence, - uint32_t flags, bool lazy, +int vmw_fence_obj_wait(struct vmw_fence_obj *fence, bool lazy, bool interruptible, unsigned long timeout) { struct vmw_private *dev_priv = fence->fman->dev_priv; long ret;

- if (likely(vmw_fence_obj_signaled(fence, flags))) + if (likely(vmw_fence_obj_signaled(fence))) return 0;

vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC); @@ -458,12 +451,12 @@ int vmw_fence_obj_wait(struct vmw_fence_obj *fence, if (interruptible) ret = wait_event_interruptible_timeout (fence->queue, - vmw_fence_obj_signaled(fence, flags), + vmw_fence_obj_signaled(fence), timeout); else ret = wait_event_timeout (fence->queue, - vmw_fence_obj_signaled(fence, flags), + vmw_fence_obj_signaled(fence), timeout);

vmw_seqno_waiter_remove(dev_priv); @@ -497,7 +490,6 @@ static void vmw_fence_destroy(struct vmw_fence_obj *fence)

int vmw_fence_create(struct vmw_fence_manager *fman, uint32_t seqno, - uint32_t mask, struct vmw_fence_obj **p_fence) { struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv); @@ -515,7 +507,7 @@ int vmw_fence_create(struct vmw_fence_manager *fman, goto out_no_object; }

- ret = vmw_fence_obj_init(fman, fence, seqno, mask, + ret = vmw_fence_obj_init(fman, fence, seqno, vmw_fence_destroy); if (unlikely(ret != 0)) goto out_err_init; @@ -559,7 +551,6 @@ static void vmw_user_fence_base_release(struct ttm_base_object **p_base) int vmw_user_fence_create(struct drm_file *file_priv, struct vmw_fence_manager *fman, uint32_t seqno, - uint32_t mask, struct vmw_fence_obj **p_fence, uint32_t *p_handle) { @@ -586,7 +577,7 @@ int vmw_user_fence_create(struct drm_file *file_priv, }

ret = vmw_fence_obj_init(fman, &ufence->fence, seqno, - mask, vmw_user_fence_destroy); + vmw_user_fence_destroy); if (unlikely(ret != 0)) { kfree(ufence); goto out_no_object; @@ -647,13 +638,12 @@ void vmw_fence_fifo_down(struct vmw_fence_manager *fman) kref_get(&fence->kref); spin_unlock_irq(&fman->lock);

- ret = vmw_fence_obj_wait(fence, fence->signal_mask, - false, false, + ret = vmw_fence_obj_wait(fence, false, false, VMW_FENCE_WAIT_TIMEOUT);

if (unlikely(ret != 0)) { list_del_init(&fence->head); - fence->signaled |= DRM_VMW_FENCE_FLAG_EXEC; + fence->signaled = 1; INIT_LIST_HEAD(&action_list); list_splice_init(&fence->seq_passed_actions, &action_list); @@ -716,14 +706,14 @@ int vmw_fence_obj_wait_ioctl(struct drm_device *dev, void *data,

timeout = jiffies; if (time_after_eq(timeout, (unsigned long)arg->kernel_cookie)) { - ret = ((vmw_fence_obj_signaled(fence, arg->flags)) ? + ret = ((vmw_fence_obj_signaled(fence)) ? 0 : -EBUSY); goto out; }

timeout = (unsigned long)arg->kernel_cookie - timeout;

- ret = vmw_fence_obj_wait(fence, arg->flags, arg->lazy, true, timeout); + ret = vmw_fence_obj_wait(fence, arg->lazy, true, timeout);

out: ttm_base_object_unref(&base); @@ -760,10 +750,10 @@ int vmw_fence_obj_signaled_ioctl(struct drm_device *dev, void *data, fence = &(container_of(base, struct vmw_user_fence, base)->fence); fman = fence->fman;

- arg->signaled = vmw_fence_obj_signaled(fence, arg->flags); + arg->signaled = vmw_fence_obj_signaled(fence); spin_lock_irq(&fman->lock);

- arg->signaled_flags = fence->signaled; + arg->signaled_flags = arg->flags; arg->passed_seqno = dev_priv->last_read_seqno; spin_unlock_irq(&fman->lock);

@@ -908,7 +898,7 @@ static void vmw_fence_obj_add_action(struct vmw_fence_obj *fence, spin_lock_irqsave(&fman->lock, irq_flags);

fman->pending_actions[action->type]++; - if (fence->signaled & DRM_VMW_FENCE_FLAG_EXEC) { + if (fence->signaled) { struct list_head action_list;

INIT_LIST_HEAD(&action_list); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h index faf2e7873860..8c18d32bd1c3 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.h @@ -56,7 +56,6 @@ struct vmw_fence_obj { struct vmw_fence_manager *fman; struct list_head head; uint32_t signaled; - uint32_t signal_mask; struct list_head seq_passed_actions; void (*destroy)(struct vmw_fence_obj *fence); wait_queue_head_t queue; @@ -74,10 +73,9 @@ vmw_fence_obj_reference(struct vmw_fence_obj *fence);

extern void vmw_fences_update(struct vmw_fence_manager *fman);

-extern bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence, - uint32_t flags); +extern bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence);

-extern int vmw_fence_obj_wait(struct vmw_fence_obj *fence, uint32_t flags, +extern int vmw_fence_obj_wait(struct vmw_fence_obj *fence, bool lazy, bool interruptible, unsigned long timeout);

@@ -85,13 +83,11 @@ extern void vmw_fence_obj_flush(struct vmw_fence_obj *fence);

extern int vmw_fence_create(struct vmw_fence_manager *fman, uint32_t seqno, - uint32_t mask, struct vmw_fence_obj **p_fence);

extern int vmw_user_fence_create(struct drm_file *file_priv, struct vmw_fence_manager *fman, uint32_t sequence, - uint32_t mask, struct vmw_fence_obj **p_fence, uint32_t *p_handle);

--- drivers/gpu/drm/nouveau/nouveau_bo.c | 48 +------- drivers/gpu/drm/nouveau/nouveau_fence.c | 24 +--- drivers/gpu/drm/nouveau/nouveau_fence.h | 2 drivers/gpu/drm/nouveau/nouveau_gem.c | 16 ++- drivers/gpu/drm/qxl/qxl_debugfs.c | 6 + drivers/gpu/drm/qxl/qxl_drv.h | 2 drivers/gpu/drm/qxl/qxl_kms.c | 1 drivers/gpu/drm/qxl/qxl_object.h | 4 - drivers/gpu/drm/qxl/qxl_release.c | 3 - drivers/gpu/drm/qxl/qxl_ttm.c | 104 ------------------ drivers/gpu/drm/radeon/radeon_cs.c | 10 +- drivers/gpu/drm/radeon/radeon_display.c | 3 - drivers/gpu/drm/radeon/radeon_object.c | 4 - drivers/gpu/drm/radeon/radeon_ttm.c | 34 ------ drivers/gpu/drm/radeon/radeon_uvd.c | 8 + drivers/gpu/drm/radeon/radeon_vm.c | 14 ++ drivers/gpu/drm/ttm/ttm_bo.c | 171 +++++++++++++++++++++--------- drivers/gpu/drm/ttm/ttm_bo_util.c | 23 +--- drivers/gpu/drm/ttm/ttm_execbuf_util.c | 10 -- drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c | 40 ------- drivers/gpu/drm/vmwgfx/vmwgfx_resource.c | 14 +- include/drm/ttm/ttm_bo_api.h | 2 include/drm/ttm/ttm_bo_driver.h | 26 ----- include/drm/ttm/ttm_execbuf_util.h | 10 +- 24 files changed, 193 insertions(+), 386 deletions(-)

diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 6e71d3dc2d87..3d8a8e18caea 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -92,13 +92,13 @@ nv10_bo_get_tile_region(struct drm_device *dev, int i)

static void nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile, - struct nouveau_fence *fence) + struct fence *fence) { struct nouveau_drm *drm = nouveau_drm(dev);

if (tile) { spin_lock(&drm->tile.lock); - tile->fence = nouveau_fence_ref(fence); + tile->fence = nouveau_fence_ref((struct nouveau_fence *)fence); tile->used = false; spin_unlock(&drm->tile.lock); } @@ -965,7 +965,8 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr, if (ret == 0) { ret = nouveau_fence_new(chan, false, &fence); if (ret == 0) { - ret = ttm_bo_move_accel_cleanup(bo, fence, + ret = ttm_bo_move_accel_cleanup(bo, + &fence->base, evict, no_wait_gpu, new_mem); @@ -1151,8 +1152,9 @@ nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, { struct nouveau_drm *drm = nouveau_bdev(bo->bdev); struct drm_device *dev = drm->dev; + struct fence *fence = reservation_object_get_excl(bo->resv);

- nv10_bo_put_tile_region(dev, *old_tile, bo->sync_obj); + nv10_bo_put_tile_region(dev, *old_tile, fence); *old_tile = new_tile; }

@@ -1423,47 +1425,14 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm) ttm_pool_unpopulate(ttm); }

-static void -nouveau_bo_fence_unref(void **sync_obj) -{ - nouveau_fence_unref((struct nouveau_fence **)sync_obj); -} - void nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence) { struct reservation_object *resv = nvbo->bo.resv;

- nouveau_bo_fence_unref(&nvbo->bo.sync_obj); - nvbo->bo.sync_obj = nouveau_fence_ref(fence); - reservation_object_add_excl_fence(resv, &fence->base); }

-static void * -nouveau_bo_fence_ref(void *sync_obj) -{ - return nouveau_fence_ref(sync_obj); -} - -static bool -nouveau_bo_fence_signalled(void *sync_obj) -{ - return nouveau_fence_done(sync_obj); -} - -static int -nouveau_bo_fence_wait(void *sync_obj, bool lazy, bool intr) -{ - return nouveau_fence_wait(sync_obj, lazy, intr); -} - -static int -nouveau_bo_fence_flush(void *sync_obj) -{ - return 0; -} - struct ttm_bo_driver nouveau_bo_driver = { .ttm_tt_create = &nouveau_ttm_tt_create, .ttm_tt_populate = &nouveau_ttm_tt_populate, @@ -1474,11 +1443,6 @@ struct ttm_bo_driver nouveau_bo_driver = { .move_notify = nouveau_bo_move_ntfy, .move = nouveau_bo_move, .verify_access = nouveau_bo_verify_access, - .sync_obj_signaled = nouveau_bo_fence_signalled, - .sync_obj_wait = nouveau_bo_fence_wait, - .sync_obj_flush = nouveau_bo_fence_flush, - .sync_obj_unref = nouveau_bo_fence_unref, - .sync_obj_ref = nouveau_bo_fence_ref, .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify, .io_mem_reserve = &nouveau_ttm_io_mem_reserve, .io_mem_free = &nouveau_ttm_io_mem_free, diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c index d24f8ce4341a..9f92ad37637d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.c +++ b/drivers/gpu/drm/nouveau/nouveau_fence.c @@ -139,17 +139,18 @@ static bool nouveau_fence_is_signaled(struct fence *f) }

void -nouveau_fence_work(struct nouveau_fence *fence, +nouveau_fence_work(struct fence *fence, void (*func)(void *), void *data) { struct nouveau_fence_work *work;

- if (fence_is_signaled(&fence->base)) + if (fence_is_signaled(fence)) goto err;

work = kmalloc(sizeof(*work), GFP_KERNEL); if (!work) { - WARN_ON(nouveau_fence_wait(fence, false, false)); + WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence, + false, false)); goto err; }

@@ -157,7 +158,7 @@ nouveau_fence_work(struct nouveau_fence *fence, work->func = func; work->data = data;

- if (fence_add_callback(&fence->base, &work->cb, nouveau_fence_work_cb) < 0) + if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0) goto err_free; return;

@@ -322,14 +323,9 @@ nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan) struct reservation_object_list *fobj; int ret = 0, i;

- fence = nvbo->bo.sync_obj; - if (fence && fence_is_signaled(fence)) { - nouveau_fence_unref((struct nouveau_fence **) - &nvbo->bo.sync_obj); - fence = NULL; - } + fence = reservation_object_get_excl(resv);

- if (fence) { + if (fence && !fence_is_signaled(fence)) { struct nouveau_fence *f = container_of(fence, struct nouveau_fence, base); @@ -345,12 +341,8 @@ nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan) if (ret) return ret;

- fence = reservation_object_get_excl(resv); - if (fence && !nouveau_local_fence(fence, chan->drm)) - ret = fence_wait(fence, true); - fobj = reservation_object_get_list(resv); - if (!fobj || ret) + if (!fobj) return ret;

for (i = 0; i < fobj->shared_count && !ret; ++i) { diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h index 1989ec22e66e..41abc8a44e3c 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.h +++ b/drivers/gpu/drm/nouveau/nouveau_fence.h @@ -26,7 +26,7 @@ void nouveau_fence_unref(struct nouveau_fence **);

int nouveau_fence_emit(struct nouveau_fence *, struct nouveau_channel *); bool nouveau_fence_done(struct nouveau_fence *); -void nouveau_fence_work(struct nouveau_fence *, void (*)(void *), void *); +void nouveau_fence_work(struct fence *, void (*)(void *), void *); int nouveau_fence_wait(struct nouveau_fence *, bool lazy, bool intr); int nouveau_fence_sync(struct nouveau_bo *, struct nouveau_channel *);

diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 2bd62f4ecf6a..0a1fee7eb211 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -100,13 +100,12 @@ static void nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) { const bool mapped = nvbo->bo.mem.mem_type != TTM_PL_SYSTEM; - struct nouveau_fence *fence = NULL; + struct fence *fence = NULL;

list_del(&vma->head);

- if (mapped) { - fence = nouveau_fence_ref(nvbo->bo.sync_obj); - } + if (mapped) + fence = reservation_object_get_excl(nvbo->bo.resv);

if (fence) { nouveau_fence_work(fence, nouveau_gem_object_delete, vma); @@ -116,7 +115,6 @@ nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) nouveau_vm_put(vma); kfree(vma); } - nouveau_fence_unref(&fence); }

void @@ -876,8 +874,12 @@ nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, ret = ttm_bo_reserve(&nvbo->bo, true, false, false, 0); if (!ret) { ret = ttm_bo_wait(&nvbo->bo, true, true, true); - if (!no_wait && ret) - fence = nouveau_fence_ref(nvbo->bo.sync_obj); + if (!no_wait && ret) { + struct fence *excl; + + excl = reservation_object_get_excl(nvbo->bo.resv); + fence = nouveau_fence_ref((struct nouveau_fence *)excl); + }

ttm_bo_unreserve(&nvbo->bo); } diff --git a/drivers/gpu/drm/qxl/qxl_debugfs.c b/drivers/gpu/drm/qxl/qxl_debugfs.c index 0d144e0646d6..a4a63fd84803 100644 --- a/drivers/gpu/drm/qxl/qxl_debugfs.c +++ b/drivers/gpu/drm/qxl/qxl_debugfs.c @@ -67,9 +67,9 @@ qxl_debugfs_buffers_info(struct seq_file *m, void *data) rel = fobj ? fobj->shared_count : 0; rcu_read_unlock();

- seq_printf(m, "size %ld, pc %d, sync obj %p, num releases %d\n", - (unsigned long)bo->gem_base.size, bo->pin_count, - bo->tbo.sync_obj, rel); + seq_printf(m, "size %ld, pc %d, num releases %d\n", + (unsigned long)bo->gem_base.size, + bo->pin_count, rel); } spin_unlock(&qdev->release_lock); return 0; diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h index d547cbdebeb4..74e2117ee0e6 100644 --- a/drivers/gpu/drm/qxl/qxl_drv.h +++ b/drivers/gpu/drm/qxl/qxl_drv.h @@ -280,9 +280,7 @@ struct qxl_device { uint8_t slot_gen_bits; uint64_t va_slot_mask;

- /* XXX: when rcu becomes available, release_lock can be killed */ spinlock_t release_lock; - spinlock_t fence_lock; struct idr release_idr; uint32_t release_seqno; spinlock_t release_idr_lock; diff --git a/drivers/gpu/drm/qxl/qxl_kms.c b/drivers/gpu/drm/qxl/qxl_kms.c index a9e7c30e92c5..7234561e09d9 100644 --- a/drivers/gpu/drm/qxl/qxl_kms.c +++ b/drivers/gpu/drm/qxl/qxl_kms.c @@ -224,7 +224,6 @@ static int qxl_device_init(struct qxl_device *qdev, idr_init(&qdev->release_idr); spin_lock_init(&qdev->release_idr_lock); spin_lock_init(&qdev->release_lock); - spin_lock_init(&qdev->fence_lock);

idr_init(&qdev->surf_id_idr); spin_lock_init(&qdev->surf_id_idr_lock); diff --git a/drivers/gpu/drm/qxl/qxl_object.h b/drivers/gpu/drm/qxl/qxl_object.h index 1edaf5768086..37af1bc0dd00 100644 --- a/drivers/gpu/drm/qxl/qxl_object.h +++ b/drivers/gpu/drm/qxl/qxl_object.h @@ -78,8 +78,8 @@ static inline int qxl_bo_wait(struct qxl_bo *bo, u32 *mem_type, } if (mem_type) *mem_type = bo->tbo.mem.mem_type; - if (bo->tbo.sync_obj) - r = ttm_bo_wait(&bo->tbo, true, true, no_wait); + + r = ttm_bo_wait(&bo->tbo, true, true, no_wait); ttm_bo_unreserve(&bo->tbo); return r; } diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c index 9731d2540a40..15158c5a5b3a 100644 --- a/drivers/gpu/drm/qxl/qxl_release.c +++ b/drivers/gpu/drm/qxl/qxl_release.c @@ -464,9 +464,6 @@ void qxl_release_fence_buffer_objects(struct qxl_release *release) bo = entry->bo; qbo = to_qxl_bo(bo);

- if (!entry->bo->sync_obj) - entry->bo->sync_obj = qbo; - reservation_object_add_shared_fence(bo->resv, &release->base); ttm_bo_add_to_lru(bo); __ttm_bo_unreserve(bo); diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c index 6230251fa5b0..99b7ee110a98 100644 --- a/drivers/gpu/drm/qxl/qxl_ttm.c +++ b/drivers/gpu/drm/qxl/qxl_ttm.c @@ -355,105 +355,6 @@ static int qxl_bo_move(struct ttm_buffer_object *bo, return ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem); }

-static bool qxl_sync_obj_signaled(void *sync_obj); - -static int qxl_sync_obj_wait(void *sync_obj, - bool lazy, bool interruptible) -{ - struct qxl_bo *bo = (struct qxl_bo *)sync_obj; - struct qxl_device *qdev = bo->gem_base.dev->dev_private; - struct reservation_object_list *fobj; - int count = 0, sc = 0, num_release = 0; - bool have_drawable_releases; - -retry: - if (sc == 0) { - if (bo->type == QXL_GEM_DOMAIN_SURFACE) - qxl_update_surface(qdev, bo); - } else if (sc >= 1) { - qxl_io_notify_oom(qdev); - } - - sc++; - - for (count = 0; count < 10; count++) { - if (qxl_sync_obj_signaled(sync_obj)) - return 0; - - if (!qxl_queue_garbage_collect(qdev, true)) - break; - } - - have_drawable_releases = false; - num_release = 0; - - spin_lock(&qdev->release_lock); - fobj = bo->tbo.resv->fence; - for (count = 0; fobj && count < fobj->shared_count; count++) { - struct qxl_release *release; - - release = container_of(fobj->shared[count], - struct qxl_release, base); - - if (fence_is_signaled(&release->base)) - continue; - - num_release++; - - if (release->type == QXL_RELEASE_DRAWABLE) - have_drawable_releases = true; - } - spin_unlock(&qdev->release_lock); - - qxl_queue_garbage_collect(qdev, true); - - if (have_drawable_releases || sc < 4) { - if (sc > 2) - /* back off */ - usleep_range(500, 1000); - if (have_drawable_releases && sc > 300) { - WARN(1, "sync obj %d still has outstanding releases %d %d %d %ld %d\n", sc, bo->surface_id, bo->is_primary, bo->pin_count, (unsigned long)bo->gem_base.size, num_release); - return -EBUSY; - } - goto retry; - } - return 0; -} - -static int qxl_sync_obj_flush(void *sync_obj) -{ - return 0; -} - -static void qxl_sync_obj_unref(void **sync_obj) -{ - *sync_obj = NULL; -} - -static void *qxl_sync_obj_ref(void *sync_obj) -{ - return sync_obj; -} - -static bool qxl_sync_obj_signaled(void *sync_obj) -{ - struct qxl_bo *qbo = (struct qxl_bo *)sync_obj; - struct qxl_device *qdev = qbo->gem_base.dev->dev_private; - struct reservation_object_list *fobj; - bool ret = true; - unsigned i; - - spin_lock(&qdev->release_lock); - fobj = qbo->tbo.resv->fence; - for (i = 0; fobj && i < fobj->shared_count; ++i) { - ret = fence_is_signaled(fobj->shared[i]); - if (!ret) - break; - } - spin_unlock(&qdev->release_lock); - return ret; -} - static void qxl_bo_move_notify(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem) { @@ -480,11 +381,6 @@ static struct ttm_bo_driver qxl_bo_driver = { .verify_access = &qxl_verify_access, .io_mem_reserve = &qxl_ttm_io_mem_reserve, .io_mem_free = &qxl_ttm_io_mem_free, - .sync_obj_signaled = &qxl_sync_obj_signaled, - .sync_obj_wait = &qxl_sync_obj_wait, - .sync_obj_flush = &qxl_sync_obj_flush, - .sync_obj_unref = &qxl_sync_obj_unref, - .sync_obj_ref = &qxl_sync_obj_ref, .move_notify = &qxl_bo_move_notify, };

diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index ae763f60c8a0..687b4fdc988e 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c @@ -228,11 +228,17 @@ static void radeon_cs_sync_rings(struct radeon_cs_parser *p) int i;

for (i = 0; i < p->nrelocs; i++) { + struct reservation_object *resv; + struct fence *fence; + if (!p->relocs[i].robj) continue;

+ resv = p->relocs[i].robj->tbo.resv; + fence = reservation_object_get_excl(resv); + radeon_semaphore_sync_to(p->ib.semaphore, - p->relocs[i].robj->tbo.sync_obj); + (struct radeon_fence *)fence); } }

@@ -402,7 +408,7 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bo

ttm_eu_fence_buffer_objects(&parser->ticket, &parser->validated, - parser->ib.fence); + &parser->ib.fence->base); } else if (backoff) { ttm_eu_backoff_reservation(&parser->ticket, &parser->validated); diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c index 3402540aa641..45bdc829bb14 100644 --- a/drivers/gpu/drm/radeon/radeon_display.c +++ b/drivers/gpu/drm/radeon/radeon_display.c @@ -481,8 +481,7 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc, goto cleanup; }

- if (new_rbo->tbo.sync_obj) - work->fence = radeon_fence_ref(new_rbo->tbo.sync_obj); + work->fence = fence_get(reservation_object_get_excl(new_rbo->tbo.resv));

radeon_bo_get_tiling_flags(new_rbo, &tiling_flags, NULL); radeon_bo_unreserve(new_rbo); diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c index 8538aebb6580..53104f80d382 100644 --- a/drivers/gpu/drm/radeon/radeon_object.c +++ b/drivers/gpu/drm/radeon/radeon_object.c @@ -736,8 +736,8 @@ int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait) return r; if (mem_type) *mem_type = bo->tbo.mem.mem_type; - if (bo->tbo.sync_obj) - r = ttm_bo_wait(&bo->tbo, true, true, no_wait); + + r = ttm_bo_wait(&bo->tbo, true, true, no_wait); ttm_bo_unreserve(&bo->tbo); return r; } diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index c8a8a5144ec1..715e29f984c1 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -265,12 +265,12 @@ static int radeon_move_blit(struct ttm_buffer_object *bo, BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0);

/* sync other rings */ - fence = bo->sync_obj; + fence = (struct radeon_fence *)reservation_object_get_excl(bo->resv); r = radeon_copy(rdev, old_start, new_start, new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE), /* GPU pages */ &fence); /* FIXME: handle copy error */ - r = ttm_bo_move_accel_cleanup(bo, (void *)fence, + r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, no_wait_gpu, new_mem); radeon_fence_unref(&fence); return r; @@ -483,31 +483,6 @@ static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_re { }

-static int radeon_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible) -{ - return radeon_fence_wait((struct radeon_fence *)sync_obj, interruptible); -} - -static int radeon_sync_obj_flush(void *sync_obj) -{ - return 0; -} - -static void radeon_sync_obj_unref(void **sync_obj) -{ - radeon_fence_unref((struct radeon_fence **)sync_obj); -} - -static void *radeon_sync_obj_ref(void *sync_obj) -{ - return radeon_fence_ref((struct radeon_fence *)sync_obj); -} - -static bool radeon_sync_obj_signaled(void *sync_obj) -{ - return radeon_fence_signaled((struct radeon_fence *)sync_obj); -} - /* * TTM backend functions. */ @@ -685,11 +660,6 @@ static struct ttm_bo_driver radeon_bo_driver = { .evict_flags = &radeon_evict_flags, .move = &radeon_bo_move, .verify_access = &radeon_verify_access, - .sync_obj_signaled = &radeon_sync_obj_signaled, - .sync_obj_wait = &radeon_sync_obj_wait, - .sync_obj_flush = &radeon_sync_obj_flush, - .sync_obj_unref = &radeon_sync_obj_unref, - .sync_obj_ref = &radeon_sync_obj_ref, .move_notify = &radeon_bo_move_notify, .fault_reserve_notify = &radeon_bo_fault_reserve_notify, .io_mem_reserve = &radeon_ttm_io_mem_reserve, diff --git a/drivers/gpu/drm/radeon/radeon_uvd.c b/drivers/gpu/drm/radeon/radeon_uvd.c index 67b2a367df40..b35655e2e35e 100644 --- a/drivers/gpu/drm/radeon/radeon_uvd.c +++ b/drivers/gpu/drm/radeon/radeon_uvd.c @@ -356,6 +356,7 @@ static int radeon_uvd_cs_msg(struct radeon_cs_parser *p, struct radeon_bo *bo, { int32_t *msg, msg_type, handle; unsigned img_size = 0; + struct fence *f; void *ptr;

int i, r; @@ -365,8 +366,9 @@ static int radeon_uvd_cs_msg(struct radeon_cs_parser *p, struct radeon_bo *bo, return -EINVAL; }

- if (bo->tbo.sync_obj) { - r = radeon_fence_wait(bo->tbo.sync_obj, false); + f = reservation_object_get_excl(bo->tbo.resv); + if (f) { + r = radeon_fence_wait((struct radeon_fence *)f, false); if (r) { DRM_ERROR("Failed waiting for UVD message (%d)!\n", r); return r; @@ -649,7 +651,7 @@ static int radeon_uvd_send_msg(struct radeon_device *rdev, r = radeon_ib_schedule(rdev, &ib, NULL); if (r) goto err; - ttm_eu_fence_buffer_objects(&ticket, &head, ib.fence); + ttm_eu_fence_buffer_objects(&ticket, &head, &ib.fence->base);

if (fence) *fence = radeon_fence_ref(ib.fence); diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index 0a0d0291af0d..e353eacb63b1 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -389,7 +389,7 @@ static int radeon_vm_clear_bo(struct radeon_device *rdev, if (r) goto error;

- ttm_eu_fence_buffer_objects(&ticket, &head, ib.fence); + ttm_eu_fence_buffer_objects(&ticket, &head, &ib.fence->base); radeon_ib_free(rdev, &ib);

return 0; @@ -658,7 +658,12 @@ int radeon_vm_update_page_directory(struct radeon_device *rdev, incr, R600_PTE_VALID);

if (ib.length_dw != 0) { - radeon_semaphore_sync_to(ib.semaphore, pd->tbo.sync_obj); + struct fence *fence; + + fence = reservation_object_get_excl(pd->tbo.resv); + radeon_semaphore_sync_to(ib.semaphore, + (struct radeon_fence *)fence); + radeon_semaphore_sync_to(ib.semaphore, vm->last_id_use); r = radeon_ib_schedule(rdev, &ib, NULL); if (r) { @@ -786,8 +791,11 @@ static void radeon_vm_update_ptes(struct radeon_device *rdev, struct radeon_bo *pt = vm->page_tables[pt_idx].bo; unsigned nptes; uint64_t pte; + struct fence *fence;

- radeon_semaphore_sync_to(ib->semaphore, pt->tbo.sync_obj); + fence = reservation_object_get_excl(pt->tbo.resv); + radeon_semaphore_sync_to(ib->semaphore, + (struct radeon_fence *)fence);

if ((addr & ~mask) == (end & ~mask)) nptes = end - addr; diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index c441ac6d546b..9d36cffad3b7 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -40,6 +40,7 @@ #include <linux/file.h> #include <linux/module.h> #include <linux/atomic.h> +#include <linux/reservation.h>

#define TTM_ASSERT_LOCKED(param) #define TTM_DEBUG(fmt, arg...) @@ -141,7 +142,6 @@ static void ttm_bo_release_list(struct kref *list_kref) BUG_ON(atomic_read(&bo->list_kref.refcount)); BUG_ON(atomic_read(&bo->kref.refcount)); BUG_ON(atomic_read(&bo->cpu_writers)); - BUG_ON(bo->sync_obj != NULL); BUG_ON(bo->mem.mm_node != NULL); BUG_ON(!list_empty(&bo->lru)); BUG_ON(!list_empty(&bo->ddestroy)); @@ -402,12 +402,30 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) ww_mutex_unlock (&bo->resv->lock); }

+static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) +{ + struct reservation_object_list *fobj; + struct fence *fence; + int i; + + fobj = reservation_object_get_list(bo->resv); + fence = reservation_object_get_excl(bo->resv); + if (fence && !fence->ops->signaled) + fence_enable_sw_signaling(fence); + + for (i = 0; fobj && i < fobj->shared_count; ++i) { + fence = rcu_dereference_protected(fobj->shared[i], + reservation_object_held(bo->resv)); + + if (!fence->ops->signaled) + fence_enable_sw_signaling(fence); + } +} + static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_global *glob = bo->glob; - struct ttm_bo_driver *driver = bdev->driver; - void *sync_obj = NULL; int put_count; int ret;

@@ -415,9 +433,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ret = __ttm_bo_reserve(bo, false, true, false, NULL);

if (!ret) { - (void) ttm_bo_wait(bo, false, false, true); - - if (!bo->sync_obj) { + if (!ttm_bo_wait(bo, false, false, true)) { put_count = ttm_bo_del_from_lru(bo);

spin_unlock(&glob->lru_lock); @@ -426,8 +442,8 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ttm_bo_list_ref_sub(bo, put_count, true);

return; - } - sync_obj = driver->sync_obj_ref(bo->sync_obj); + } else + ttm_bo_flush_all_fences(bo);

/* * Make NO_EVICT bos immediately available to @@ -446,14 +462,70 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) list_add_tail(&bo->ddestroy, &bdev->ddestroy); spin_unlock(&glob->lru_lock);

- if (sync_obj) { - driver->sync_obj_flush(sync_obj); - driver->sync_obj_unref(&sync_obj); - } schedule_delayed_work(&bdev->wq, ((HZ / 100) < 1) ? 1 : HZ / 100); }

+static int ttm_bo_unreserve_and_wait(struct ttm_buffer_object *bo, + bool interruptible) +{ + struct ttm_bo_global *glob = bo->glob; + struct reservation_object_list *fobj; + struct fence *excl = NULL; + struct fence **shared = NULL; + u32 shared_count = 0, i; + int ret = 0; + + fobj = reservation_object_get_list(bo->resv); + if (fobj && fobj->shared_count) { + shared = kmalloc(sizeof(*shared) * fobj->shared_count, + GFP_KERNEL); + + if (!shared) { + ret = -ENOMEM; + __ttm_bo_unreserve(bo); + spin_unlock(&glob->lru_lock); + return ret; + } + + for (i = 0; i < fobj->shared_count; ++i) { + if (!fence_is_signaled(fobj->shared[i])) { + fence_get(fobj->shared[i]); + shared[shared_count++] = fobj->shared[i]; + } + } + if (!shared_count) { + kfree(shared); + shared = NULL; + } + } + + excl = reservation_object_get_excl(bo->resv); + if (excl && !fence_is_signaled(excl)) + fence_get(excl); + else + excl = NULL; + + __ttm_bo_unreserve(bo); + spin_unlock(&glob->lru_lock); + + if (excl) { + ret = fence_wait(excl, interruptible); + fence_put(excl); + } + + if (shared_count > 0) { + for (i = 0; i < shared_count; ++i) { + if (!ret) + ret = fence_wait(shared[i], interruptible); + fence_put(shared[i]); + } + kfree(shared); + } + + return ret; +} + /** * function ttm_bo_cleanup_refs_and_unlock * If bo idle, remove from delayed- and lru lists, and unref. @@ -470,8 +542,6 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, bool interruptible, bool no_wait_gpu) { - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_driver *driver = bdev->driver; struct ttm_bo_global *glob = bo->glob; int put_count; int ret; @@ -479,20 +549,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, ret = ttm_bo_wait(bo, false, false, true);

if (ret && !no_wait_gpu) { - void *sync_obj; - - /* - * Take a reference to the fence and unreserve, - * at this point the buffer should be dead, so - * no new sync objects can be attached. - */ - sync_obj = driver->sync_obj_ref(bo->sync_obj); - - __ttm_bo_unreserve(bo); - spin_unlock(&glob->lru_lock); - - ret = driver->sync_obj_wait(sync_obj, false, interruptible); - driver->sync_obj_unref(&sync_obj); + ret = ttm_bo_unreserve_and_wait(bo, interruptible); if (ret) return ret;

@@ -1513,41 +1570,51 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)

EXPORT_SYMBOL(ttm_bo_unmap_virtual);

- int ttm_bo_wait(struct ttm_buffer_object *bo, bool lazy, bool interruptible, bool no_wait) { - struct ttm_bo_driver *driver = bo->bdev->driver; - void *sync_obj; - int ret = 0; - - lockdep_assert_held(&bo->resv->lock.base); + struct reservation_object_list *fobj; + struct reservation_object *resv; + struct fence *excl; + long timeout = 15 * HZ; + int i;

- if (likely(bo->sync_obj == NULL)) - return 0; + resv = bo->resv; + fobj = reservation_object_get_list(resv); + excl = reservation_object_get_excl(resv); + if (excl) { + if (!fence_is_signaled(excl)) { + if (no_wait) + return -EBUSY;

- if (bo->sync_obj) { - if (driver->sync_obj_signaled(bo->sync_obj)) { - driver->sync_obj_unref(&bo->sync_obj); - clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); - return 0; + timeout = fence_wait_timeout(excl, + interruptible, timeout); } + }

- if (no_wait) - return -EBUSY; + for (i = 0; fobj && timeout > 0 && i < fobj->shared_count; ++i) { + struct fence *fence; + fence = rcu_dereference_protected(fobj->shared[i], + reservation_object_held(resv));

- sync_obj = driver->sync_obj_ref(bo->sync_obj); - ret = driver->sync_obj_wait(sync_obj, - lazy, interruptible); + if (!fence_is_signaled(fence)) { + if (no_wait) + return -EBUSY;

- if (likely(ret == 0)) { - clear_bit(TTM_BO_PRIV_FLAG_MOVING, - &bo->priv_flags); - driver->sync_obj_unref(&bo->sync_obj); + timeout = fence_wait_timeout(fence, + interruptible, timeout); } - driver->sync_obj_unref(&sync_obj); } - return ret; + + if (timeout < 0) + return timeout; + + if (timeout == 0) + return -EBUSY; + + reservation_object_add_excl_fence(resv, NULL); + clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); + return 0; } EXPORT_SYMBOL(ttm_bo_wait);

diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 495aebf0f9c3..824af90cbe31 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -37,6 +37,7 @@ #include <linux/slab.h> #include <linux/vmalloc.h> #include <linux/module.h> +#include <linux/reservation.h>

void ttm_bo_free_old_node(struct ttm_buffer_object *bo) { @@ -444,8 +445,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, struct ttm_buffer_object **new_obj) { struct ttm_buffer_object *fbo; - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_driver *driver = bdev->driver; int ret;

fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); @@ -466,10 +465,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, drm_vma_node_reset(&fbo->vma_node); atomic_set(&fbo->cpu_writers, 0);

- if (bo->sync_obj) - fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); - else - fbo->sync_obj = NULL; kref_init(&fbo->list_kref); kref_init(&fbo->kref); fbo->destroy = &ttm_transfered_destroy; @@ -642,28 +637,20 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) EXPORT_SYMBOL(ttm_bo_kunmap);

int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, - void *sync_obj, + struct fence *fence, bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem) { struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_driver *driver = bdev->driver; struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; struct ttm_mem_reg *old_mem = &bo->mem; int ret; struct ttm_buffer_object *ghost_obj; - void *tmp_obj = NULL;

- if (bo->sync_obj) { - tmp_obj = bo->sync_obj; - bo->sync_obj = NULL; - } - bo->sync_obj = driver->sync_obj_ref(sync_obj); + reservation_object_add_excl_fence(bo->resv, fence); if (evict) { ret = ttm_bo_wait(bo, false, false, false); - if (tmp_obj) - driver->sync_obj_unref(&tmp_obj); if (ret) return ret;

@@ -684,13 +671,13 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, */

set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); - if (tmp_obj) - driver->sync_obj_unref(&tmp_obj);

ret = ttm_buffer_object_transfer(bo, &ghost_obj); if (ret) return ret;

+ reservation_object_add_excl_fence(ghost_obj->resv, fence); + /** * If we're not moving to fixed memory, the TTM object * needs to stay alive. Otherwhise hang it on the ghost diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index 108730e9147b..adafc0f8ec06 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -163,7 +163,7 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, EXPORT_SYMBOL(ttm_eu_reserve_buffers);

void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, - struct list_head *list, void *sync_obj) + struct list_head *list, struct fence *fence) { struct ttm_validate_buffer *entry; struct ttm_buffer_object *bo; @@ -183,18 +183,12 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket,

list_for_each_entry(entry, list, head) { bo = entry->bo; - entry->old_sync_obj = bo->sync_obj; - bo->sync_obj = driver->sync_obj_ref(sync_obj); + reservation_object_add_excl_fence(bo->resv, fence); ttm_bo_add_to_lru(bo); __ttm_bo_unreserve(bo); } spin_unlock(&glob->lru_lock); if (ticket) ww_acquire_fini(ticket); - - list_for_each_entry(entry, list, head) { - if (entry->old_sync_obj) - driver->sync_obj_unref(&entry->old_sync_obj); - } } EXPORT_SYMBOL(ttm_eu_fence_buffer_objects); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c index f15718cc631d..656c88485e14 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c @@ -768,41 +768,6 @@ static int vmw_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) }

/** - * FIXME: We're using the old vmware polling method to sync. - * Do this with fences instead. - */ - -static void *vmw_sync_obj_ref(void *sync_obj) -{ - - return (void *) - vmw_fence_obj_reference((struct vmw_fence_obj *) sync_obj); -} - -static void vmw_sync_obj_unref(void **sync_obj) -{ - vmw_fence_obj_unreference((struct vmw_fence_obj **) sync_obj); -} - -static int vmw_sync_obj_flush(void *sync_obj) -{ - vmw_fence_obj_flush((struct vmw_fence_obj *) sync_obj); - return 0; -} - -static bool vmw_sync_obj_signaled(void *sync_obj) -{ - return vmw_fence_obj_signaled((struct vmw_fence_obj *) sync_obj); -} - -static int vmw_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible) -{ - return vmw_fence_obj_wait((struct vmw_fence_obj *) sync_obj, - lazy, interruptible, - VMW_FENCE_WAIT_TIMEOUT); -} - -/** * vmw_move_notify - TTM move_notify_callback * * @bo: The TTM buffer object about to move. @@ -839,11 +804,6 @@ struct ttm_bo_driver vmw_bo_driver = { .evict_flags = vmw_evict_flags, .move = NULL, .verify_access = vmw_verify_access, - .sync_obj_signaled = vmw_sync_obj_signaled, - .sync_obj_wait = vmw_sync_obj_wait, - .sync_obj_flush = vmw_sync_obj_flush, - .sync_obj_unref = vmw_sync_obj_unref, - .sync_obj_ref = vmw_sync_obj_ref, .move_notify = vmw_move_notify, .swap_notify = vmw_swap_notify, .fault_reserve_notify = &vmw_ttm_fault_reserve_notify, diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c index 294a0a3e48cb..f0a689ba9685 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c @@ -1419,22 +1419,16 @@ void vmw_fence_single_bo(struct ttm_buffer_object *bo, struct vmw_fence_obj *fence) { struct ttm_bo_device *bdev = bo->bdev; - struct vmw_fence_obj *old_fence_obj; + struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev);

if (fence == NULL) { vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL); + reservation_object_add_excl_fence(bo->resv, &fence->base); + fence_put(&fence->base); } else - vmw_fence_obj_reference(fence); - - reservation_object_add_excl_fence(bo->resv, &fence->base); - - old_fence_obj = bo->sync_obj; - bo->sync_obj = fence; - - if (old_fence_obj) - vmw_fence_obj_unreference(&old_fence_obj); + reservation_object_add_excl_fence(bo->resv, &fence->base); }

/** diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index 67df9d7c06cc..3b630f3153d0 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -163,7 +163,6 @@ struct ttm_tt; * @lru: List head for the lru list. * @ddestroy: List head for the delayed destroy list. * @swap: List head for swap LRU list. - * @sync_obj: Pointer to a synchronization object. * @priv_flags: Flags describing buffer object internal state. * @vma_node: Address space manager node. * @offset: The current GPU offset, which can have different meanings @@ -230,7 +229,6 @@ struct ttm_buffer_object { * Members protected by a bo reservation. */

- void *sync_obj; unsigned long priv_flags;

struct drm_vma_offset_node vma_node; diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index ff725df48595..9792cd047b8c 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -311,11 +311,6 @@ struct ttm_mem_type_manager { * @move: Callback for a driver to hook in accelerated functions to * move a buffer. * If set to NULL, a potentially slow memcpy() move is used. - * @sync_obj_signaled: See ttm_fence_api.h - * @sync_obj_wait: See ttm_fence_api.h - * @sync_obj_flush: See ttm_fence_api.h - * @sync_obj_unref: See ttm_fence_api.h - * @sync_obj_ref: See ttm_fence_api.h */

struct ttm_bo_driver { @@ -417,23 +412,6 @@ struct ttm_bo_driver { int (*verify_access) (struct ttm_buffer_object *bo, struct file *filp);

- /** - * In case a driver writer dislikes the TTM fence objects, - * the driver writer can replace those with sync objects of - * his / her own. If it turns out that no driver writer is - * using these. I suggest we remove these hooks and plug in - * fences directly. The bo driver needs the following functionality: - * See the corresponding functions in the fence object API - * documentation. - */ - - bool (*sync_obj_signaled) (void *sync_obj); - int (*sync_obj_wait) (void *sync_obj, - bool lazy, bool interruptible); - int (*sync_obj_flush) (void *sync_obj); - void (*sync_obj_unref) (void **sync_obj); - void *(*sync_obj_ref) (void *sync_obj); - /* hook to notify driver about a driver move so it * can do tiling things */ void (*move_notify)(struct ttm_buffer_object *bo, @@ -1021,7 +999,7 @@ extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo); * ttm_bo_move_accel_cleanup. * * @bo: A pointer to a struct ttm_buffer_object. - * @sync_obj: A sync object that signals when moving is complete. + * @fence: A fence object that signals when moving is complete. * @evict: This is an evict move. Don't return until the buffer is idle. * @no_wait_gpu: Return immediately if the GPU is busy. * @new_mem: struct ttm_mem_reg indicating where to move. @@ -1035,7 +1013,7 @@ extern void ttm_bo_free_old_node(struct ttm_buffer_object *bo); */

extern int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, - void *sync_obj, + struct fence *fence, bool evict, bool no_wait_gpu, struct ttm_mem_reg *new_mem); /** diff --git a/include/drm/ttm/ttm_execbuf_util.h b/include/drm/ttm/ttm_execbuf_util.h index 8490cb8ee0d8..ff11a424f752 100644 --- a/include/drm/ttm/ttm_execbuf_util.h +++ b/include/drm/ttm/ttm_execbuf_util.h @@ -39,16 +39,11 @@ * * @head: list head for thread-private list. * @bo: refcounted buffer object pointer. - * @reserved: Indicates whether @bo has been reserved for validation. - * @removed: Indicates whether @bo has been removed from lru lists. - * @put_count: Number of outstanding references on bo::list_kref. - * @old_sync_obj: Pointer to a sync object about to be unreferenced */

struct ttm_validate_buffer { struct list_head head; struct ttm_buffer_object *bo; - void *old_sync_obj; };

/** @@ -100,7 +95,7 @@ extern int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, * * @ticket: ww_acquire_ctx from reserve call * @list: thread private list of ttm_validate_buffer structs. - * @sync_obj: The new sync object for the buffers. + * @fence: The new exclusive fence for the buffers. * * This function should be called when command submission is complete, and * it will add a new sync object to bos pointed to by entries on @list. @@ -109,6 +104,7 @@ extern int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, */

extern void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, - struct list_head *list, void *sync_obj); + struct list_head *list, + struct fence *fence);

#endif

Signed-off-by: Maarten Lankhorst maarten.lankhorst@canonical.com --- drivers/gpu/drm/radeon/radeon_gem.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c index d09650c1d720..7ba883843668 100644 --- a/drivers/gpu/drm/radeon/radeon_gem.c +++ b/drivers/gpu/drm/radeon/radeon_gem.c @@ -107,9 +107,12 @@ static int radeon_gem_set_domain(struct drm_gem_object *gobj, } if (domain == RADEON_GEM_DOMAIN_CPU) { /* Asking for cpu access wait for object idle */ - r = radeon_bo_wait(robj, NULL, false); - if (r) { - printk(KERN_ERR "Failed to wait for object !\n"); + r = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true, 30 * HZ); + if (!r) + r = -EBUSY; + + if (r < 0 && r != -EINTR) { + printk(KERN_ERR "Failed to wait for object: %i\n", r); return r; } } @@ -357,14 +360,20 @@ int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data, struct drm_radeon_gem_wait_idle *args = data; struct drm_gem_object *gobj; struct radeon_bo *robj; - int r; + int r = 0; + long ret;

gobj = drm_gem_object_lookup(dev, filp, args->handle); if (gobj == NULL) { return -ENOENT; } robj = gem_to_radeon_bo(gobj); - r = radeon_bo_wait(robj, NULL, false); + ret = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true, 30 * HZ); + if (ret == 0) + r = -EBUSY; + else if (ret < 0) + r = ret; + /* callback hw specific functions if any */ if (rdev->asic->ioctl_wait_idle) robj->rdev->asic->ioctl_wait_idle(rdev, robj);

Signed-off-by: Maarten Lankhorst maarten.lankhorst@canonical.com --- drivers/gpu/drm/ttm/ttm_bo.c | 76 +++++++----------------------------------- 1 file changed, 13 insertions(+), 63 deletions(-)

diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 9d36cffad3b7..d119ae4419d4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -466,66 +466,6 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ((HZ / 100) < 1) ? 1 : HZ / 100); }

-static int ttm_bo_unreserve_and_wait(struct ttm_buffer_object *bo, - bool interruptible) -{ - struct ttm_bo_global *glob = bo->glob; - struct reservation_object_list *fobj; - struct fence *excl = NULL; - struct fence **shared = NULL; - u32 shared_count = 0, i; - int ret = 0; - - fobj = reservation_object_get_list(bo->resv); - if (fobj && fobj->shared_count) { - shared = kmalloc(sizeof(*shared) * fobj->shared_count, - GFP_KERNEL); - - if (!shared) { - ret = -ENOMEM; - __ttm_bo_unreserve(bo); - spin_unlock(&glob->lru_lock); - return ret; - } - - for (i = 0; i < fobj->shared_count; ++i) { - if (!fence_is_signaled(fobj->shared[i])) { - fence_get(fobj->shared[i]); - shared[shared_count++] = fobj->shared[i]; - } - } - if (!shared_count) { - kfree(shared); - shared = NULL; - } - } - - excl = reservation_object_get_excl(bo->resv); - if (excl && !fence_is_signaled(excl)) - fence_get(excl); - else - excl = NULL; - - __ttm_bo_unreserve(bo); - spin_unlock(&glob->lru_lock); - - if (excl) { - ret = fence_wait(excl, interruptible); - fence_put(excl); - } - - if (shared_count > 0) { - for (i = 0; i < shared_count; ++i) { - if (!ret) - ret = fence_wait(shared[i], interruptible); - fence_put(shared[i]); - } - kfree(shared); - } - - return ret; -} - /** * function ttm_bo_cleanup_refs_and_unlock * If bo idle, remove from delayed- and lru lists, and unref. @@ -549,9 +489,19 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, ret = ttm_bo_wait(bo, false, false, true);

if (ret && !no_wait_gpu) { - ret = ttm_bo_unreserve_and_wait(bo, interruptible); - if (ret) - return ret; + long lret; + ww_mutex_unlock(&bo->resv->lock); + spin_unlock(&glob->lru_lock); + + lret = reservation_object_wait_timeout_rcu(bo->resv, + true, + interruptible, + 30 * HZ); + + if (lret < 0) + return lret; + else if (lret == 0) + return -EBUSY;

spin_lock(&glob->lru_lock); ret = __ttm_bo_reserve(bo, false, true, false, NULL);