From: Russell King rmk+kernel@arm.linux.org.uk
Add support for mapping userspace memory to the GPU. This is useful for cases where we have some malloc()'d memory which we wish to pass to the GPU or shmem memory via Xv, and wish to avoid the overhead of an additional memcpy(), especially as is is expensive to memcpy()ing a 1080p frame.
This is mostly taken from the 3.17 i915 userptr implementation, except we solve the held-mm problem in (imho) a nicer way, and we also avoid excessive spinning with -EAGAIN waiting for the queued work to run.
Signed-off-by: Russell King rmk+kernel@arm.linux.org.uk --- drivers/staging/etnaviv/etnaviv_drv.c | 30 +++++ drivers/staging/etnaviv/etnaviv_drv.h | 2 + drivers/staging/etnaviv/etnaviv_gem.c | 195 +++++++++++++++++++++++++++ drivers/staging/etnaviv/etnaviv_gem.h | 9 ++ drivers/staging/etnaviv/etnaviv_gem_submit.c | 9 ++ include/uapi/drm/etnaviv_drm.h | 13 +- 6 files changed, 257 insertions(+), 1 deletion(-)
diff --git a/drivers/staging/etnaviv/etnaviv_drv.c b/drivers/staging/etnaviv/etnaviv_drv.c index 2b6800a782bb..5f386a7045ae 100644 --- a/drivers/staging/etnaviv/etnaviv_drv.c +++ b/drivers/staging/etnaviv/etnaviv_drv.c @@ -454,6 +454,35 @@ static int etnaviv_ioctl_wait_fence(struct drm_device *dev, void *data, &TS(args->timeout)); }
+static int etnaviv_ioctl_gem_userptr(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct drm_etnaviv_gem_userptr *args = data; + int access; + + if (args->flags & ~(ETNA_USERPTR_READ|ETNA_USERPTR_WRITE) || + args->flags == 0) + return -EINVAL; + + if (offset_in_page(args->user_ptr | args->user_size) || + (uintptr_t)args->user_ptr != args->user_ptr || + (uint32_t)args->user_size != args->user_size) + return -EINVAL; + + if (args->flags & ETNA_USERPTR_WRITE) + access = VERIFY_WRITE; + else + access = VERIFY_READ; + + if (!access_ok(access, (void __user *)(unsigned long)args->user_ptr, + args->user_size)) + return -EFAULT; + + return etnaviv_gem_new_userptr(dev, file, args->user_ptr, + args->user_size, args->flags, + &args->handle); +} + static const struct drm_ioctl_desc etnaviv_ioctls[] = { #define ETNA_IOCTL(n, func, flags) \ DRM_IOCTL_DEF_DRV(ETNAVIV_##n, etnaviv_ioctl_##func, flags) @@ -464,6 +493,7 @@ static const struct drm_ioctl_desc etnaviv_ioctls[] = { ETNA_IOCTL(GEM_CPU_FINI, gem_cpu_fini, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW), ETNA_IOCTL(GEM_SUBMIT, gem_submit, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW), ETNA_IOCTL(WAIT_FENCE, wait_fence, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW), + ETNA_IOCTL(GEM_USERPTR, gem_userptr, DRM_UNLOCKED|DRM_RENDER_ALLOW), };
static const struct vm_operations_struct vm_ops = { diff --git a/drivers/staging/etnaviv/etnaviv_drv.h b/drivers/staging/etnaviv/etnaviv_drv.h index 79f22d15fc63..59aa4666d2cc 100644 --- a/drivers/staging/etnaviv/etnaviv_drv.h +++ b/drivers/staging/etnaviv/etnaviv_drv.h @@ -107,6 +107,8 @@ int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file, uint32_t size, uint32_t flags, uint32_t *handle); struct drm_gem_object *etnaviv_gem_new(struct drm_device *dev, uint32_t size, uint32_t flags); +int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file, + uintptr_t ptr, uint32_t size, uint32_t flags, uint32_t *handle); u32 etnaviv_buffer_init(struct etnaviv_gpu *gpu); void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event, struct etnaviv_gem_submit *submit); diff --git a/drivers/staging/etnaviv/etnaviv_gem.c b/drivers/staging/etnaviv/etnaviv_gem.c index ad8fa71ea920..4e28c57b2409 100644 --- a/drivers/staging/etnaviv/etnaviv_gem.c +++ b/drivers/staging/etnaviv/etnaviv_gem.c @@ -694,3 +694,198 @@ int etnaviv_gem_new_private(struct drm_device *dev, size_t size, uint32_t flags,
return 0; } + +struct get_pages_work { + struct work_struct work; + struct mm_struct *mm; + struct task_struct *task; + struct etnaviv_gem_object *etnaviv_obj; +}; + +static struct page **etnaviv_gem_userptr_do_get_pages( + struct etnaviv_gem_object *etnaviv_obj, struct mm_struct *mm, struct task_struct *task) +{ + int ret, pinned, npages = etnaviv_obj->base.size >> PAGE_SHIFT; + struct page **pvec; + uintptr_t ptr; + + pvec = drm_malloc_ab(npages, sizeof(struct page *)); + if (!pvec) + return ERR_PTR(-ENOMEM); + + pinned = 0; + ptr = etnaviv_obj->userptr.ptr; + + down_read(&mm->mmap_sem); + while (pinned < npages) { + ret = get_user_pages(task, mm, ptr, npages - pinned, + !etnaviv_obj->userptr.ro, 0, + pvec + pinned, NULL); + if (ret < 0) + break; + + ptr += ret * PAGE_SIZE; + pinned += ret; + } + up_read(&mm->mmap_sem); + + if (ret < 0) { + release_pages(pvec, pinned, 0); + drm_free_large(pvec); + return ERR_PTR(ret); + } + + return pvec; +} + +static void __etnaviv_gem_userptr_get_pages(struct work_struct *_work) +{ + struct get_pages_work *work = container_of(_work, typeof(*work), work); + struct etnaviv_gem_object *etnaviv_obj = work->etnaviv_obj; + struct drm_device *dev = etnaviv_obj->base.dev; + struct page **pvec; + + pvec = etnaviv_gem_userptr_do_get_pages(etnaviv_obj, work->mm, work->task); + + mutex_lock(&dev->struct_mutex); + if (IS_ERR(pvec)) { + etnaviv_obj->userptr.work = ERR_CAST(pvec); + } else { + etnaviv_obj->userptr.work = NULL; + etnaviv_obj->pages = pvec; + } + + drm_gem_object_unreference(&etnaviv_obj->base); + mutex_unlock(&dev->struct_mutex); + + mmput(work->mm); + put_task_struct(work->task); + kfree(work); +} + +static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj) +{ + struct etnaviv_drm_private *priv; + struct page **pvec = NULL; + struct get_pages_work *work; + struct mm_struct *mm; + int ret, pinned, npages = etnaviv_obj->base.size >> PAGE_SHIFT; + + if (etnaviv_obj->userptr.work) { + if (IS_ERR(etnaviv_obj->userptr.work)) { + ret = PTR_ERR(etnaviv_obj->userptr.work); + etnaviv_obj->userptr.work = NULL; + } else { + ret = -EAGAIN; + } + return ret; + } + + mm = get_task_mm(etnaviv_obj->userptr.task); + pinned = 0; + if (mm == current->mm) { + pvec = drm_malloc_ab(npages, sizeof(struct page *)); + if (!pvec) { + mmput(mm); + return -ENOMEM; + } + + pinned = __get_user_pages_fast(etnaviv_obj->userptr.ptr, npages, + !etnaviv_obj->userptr.ro, pvec); + if (pinned < 0) { + drm_free_large(pvec); + mmput(mm); + return pinned; + } + + if (pinned == npages) { + etnaviv_obj->pages = pvec; + mmput(mm); + return 0; + } + } + + release_pages(pvec, pinned, 0); + drm_free_large(pvec); + + work = kmalloc(sizeof(*work), GFP_KERNEL); + if (!work) { + mmput(mm); + return -ENOMEM; + } + + get_task_struct(current); + drm_gem_object_reference(&etnaviv_obj->base); + + work->mm = mm; + work->task = current; + work->etnaviv_obj = etnaviv_obj; + + etnaviv_obj->userptr.work = &work->work; + INIT_WORK(&work->work, __etnaviv_gem_userptr_get_pages); + + priv = etnaviv_obj->base.dev->dev_private; + queue_work(priv->wq, &work->work); + + return -EAGAIN; +} + +static void etnaviv_gem_userptr_release(struct etnaviv_gem_object *etnaviv_obj) +{ + if (etnaviv_obj->sgt) { + /* + * For non-cached buffers, ensure the new pages are clean + * because display controller, GPU, etc. are not coherent: + */ + if (etnaviv_obj->flags & (ETNA_BO_WC|ETNA_BO_CACHED)) + dma_unmap_sg(etnaviv_obj->base.dev->dev, + etnaviv_obj->sgt->sgl, + etnaviv_obj->sgt->nents, + DMA_BIDIRECTIONAL); + sg_free_table(etnaviv_obj->sgt); + kfree(etnaviv_obj->sgt); + } + if (etnaviv_obj->pages) { + int npages = etnaviv_obj->base.size >> PAGE_SHIFT; + + release_pages(etnaviv_obj->pages, npages, 0); + drm_free_large(etnaviv_obj->pages); + } + put_task_struct(etnaviv_obj->userptr.task); +} + +static const struct etnaviv_gem_ops etnaviv_gem_userptr_ops = { + .get_pages = etnaviv_gem_userptr_get_pages, + .release = etnaviv_gem_userptr_release, +}; + +int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file, + uintptr_t ptr, uint32_t size, uint32_t flags, uint32_t *handle) +{ + struct etnaviv_gem_object *etnaviv_obj; + int ret; + + ret = mutex_lock_interruptible(&dev->struct_mutex); + if (ret) + return ret; + + ret = etnaviv_gem_new_private(dev, size, ETNA_BO_CACHED, &etnaviv_obj); + if (ret == 0) { + etnaviv_obj->ops = &etnaviv_gem_userptr_ops; + etnaviv_obj->userptr.ptr = ptr; + etnaviv_obj->userptr.task = current; + etnaviv_obj->userptr.ro = !(flags & ETNA_USERPTR_WRITE); + get_task_struct(current); + } + mutex_unlock(&dev->struct_mutex); + + if (ret) + return ret; + + ret = drm_gem_handle_create(file, &etnaviv_obj->base, handle); + + /* drop reference from allocate - handle holds it now */ + drm_gem_object_unreference_unlocked(&etnaviv_obj->base); + + return ret; +} diff --git a/drivers/staging/etnaviv/etnaviv_gem.h b/drivers/staging/etnaviv/etnaviv_gem.h index bbfcbd7557fe..add616338a9f 100644 --- a/drivers/staging/etnaviv/etnaviv_gem.h +++ b/drivers/staging/etnaviv/etnaviv_gem.h @@ -23,6 +23,13 @@
struct etnaviv_gem_ops;
+struct etnaviv_gem_userptr { + uintptr_t ptr; + struct task_struct *task; + struct work_struct *work; + bool ro; +}; + struct etnaviv_gem_object { struct drm_gem_object base; const struct etnaviv_gem_ops *ops; @@ -64,6 +71,8 @@ struct etnaviv_gem_object {
/* for buffer manipulation during submit */ u32 offset; + + struct etnaviv_gem_userptr userptr; }; #define to_etnaviv_bo(x) container_of(x, struct etnaviv_gem_object, base)
diff --git a/drivers/staging/etnaviv/etnaviv_gem_submit.c b/drivers/staging/etnaviv/etnaviv_gem_submit.c index af3718465ea1..bbe2171b8eb4 100644 --- a/drivers/staging/etnaviv/etnaviv_gem_submit.c +++ b/drivers/staging/etnaviv/etnaviv_gem_submit.c @@ -435,5 +435,14 @@ out: if (submit) submit_cleanup(submit, !!ret); mutex_unlock(&dev->struct_mutex); + + /* + * If we're returning -EAGAIN, it could be due to the userptr code + * wanting to run its workqueue outside of the struct_mutex. + * Flush our workqueue to ensure that it is run in a timely manner. + */ + if (ret == -EAGAIN) + flush_workqueue(priv->wq); + return ret; } diff --git a/include/uapi/drm/etnaviv_drm.h b/include/uapi/drm/etnaviv_drm.h index 9654021017fd..a4c109ffbea4 100644 --- a/include/uapi/drm/etnaviv_drm.h +++ b/include/uapi/drm/etnaviv_drm.h @@ -202,6 +202,15 @@ struct drm_etnaviv_wait_fence { struct drm_etnaviv_timespec timeout; /* in */ };
+#define ETNA_USERPTR_READ 0x01 +#define ETNA_USERPTR_WRITE 0x02 +struct drm_etnaviv_gem_userptr { + uint64_t user_ptr; /* in, page aligned user pointer */ + uint64_t user_size; /* in, page aligned user size */ + uint32_t flags; /* in, flags */ + uint32_t handle; /* out, non-zero handle */ +}; + #define DRM_ETNAVIV_GET_PARAM 0x00 /* placeholder: #define DRM_MSM_SET_PARAM 0x01 @@ -212,7 +221,8 @@ struct drm_etnaviv_wait_fence { #define DRM_ETNAVIV_GEM_CPU_FINI 0x05 #define DRM_ETNAVIV_GEM_SUBMIT 0x06 #define DRM_ETNAVIV_WAIT_FENCE 0x07 -#define DRM_ETNAVIV_NUM_IOCTLS 0x08 +#define DRM_ETNAVIV_GEM_USERPTR 0x08 +#define DRM_ETNAVIV_NUM_IOCTLS 0x09
#define DRM_IOCTL_ETNAVIV_GET_PARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_ETNAVIV_GET_PARAM, struct drm_etnaviv_param) #define DRM_IOCTL_ETNAVIV_GEM_NEW DRM_IOWR(DRM_COMMAND_BASE + DRM_ETNAVIV_GEM_NEW, struct drm_etnaviv_gem_new) @@ -221,5 +231,6 @@ struct drm_etnaviv_wait_fence { #define DRM_IOCTL_ETNAVIV_GEM_CPU_FINI DRM_IOW(DRM_COMMAND_BASE + DRM_ETNAVIV_GEM_CPU_FINI, struct drm_etnaviv_gem_cpu_fini) #define DRM_IOCTL_ETNAVIV_GEM_SUBMIT DRM_IOWR(DRM_COMMAND_BASE + DRM_ETNAVIV_GEM_SUBMIT, struct drm_etnaviv_gem_submit) #define DRM_IOCTL_ETNAVIV_WAIT_FENCE DRM_IOW(DRM_COMMAND_BASE + DRM_ETNAVIV_WAIT_FENCE, struct drm_etnaviv_wait_fence) +#define DRM_IOCTL_ETNAVIV_GEM_USERPTR DRM_IOWR(DRM_COMMAND_BASE + DRM_ETNAVIV_GEM_USERPTR, struct drm_etnaviv_gem_userptr)
#endif /* __ETNAVIV_DRM_H__ */