This adds 3 more functions to deal with rcu.
reservation_object_wait_timeout_rcu() will wait on all fences of the reservation_object, without obtaining the ww_mutex.
reservation_object_test_signaled_rcu() will test if all fences of the reservation_object are signaled without using the ww_mutex.
reservation_object_get_excl() is added because touching the fence_excl member directly will trigger a sparse warning.
Signed-off-by: Maarten Lankhorst maarten.lankhorst@canonical.com --- drivers/base/dma-buf.c | 46 +++++++++++-- drivers/base/reservation.c | 147 +++++++++++++++++++++++++++++++++++++++++-- include/linux/fence.h | 22 ++++++ include/linux/reservation.h | 40 ++++++++---- 4 files changed, 224 insertions(+), 31 deletions(-)
diff --git a/drivers/base/dma-buf.c b/drivers/base/dma-buf.c index d89a98d2c37b..fc2d7546b8b0 100644 --- a/drivers/base/dma-buf.c +++ b/drivers/base/dma-buf.c @@ -151,14 +151,22 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll) if (!events) return 0;
- ww_mutex_lock(&resv->lock, NULL); + rcu_read_lock();
- fobj = resv->fence; - if (!fobj) - goto out; + fobj = rcu_dereference(resv->fence); + if (fobj) { + shared_count = ACCESS_ONCE(fobj->shared_count); + smp_mb(); /* shared_count needs transitivity wrt fence_excl */ + } else + shared_count = 0; + fence_excl = rcu_dereference(resv->fence_excl);
- shared_count = fobj->shared_count; - fence_excl = resv->fence_excl; + /* + * This would have needed a smp_read_barrier_depends() + * because shared_count needs to be read before shared[i], but + * spin_lock_irq and spin_unlock_irq provide even stronger + * guarantees. + */
if (fence_excl && (!(events & POLLOUT) || shared_count == 0)) { struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_excl; @@ -176,14 +184,20 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll) spin_unlock_irq(&dmabuf->poll.lock);
if (events & pevents) { - if (!fence_add_callback(fence_excl, &dcb->cb, + if (!fence_get_rcu(fence_excl)) { + /* force a recheck */ + events &= ~pevents; + dma_buf_poll_cb(NULL, &dcb->cb); + } else if (!fence_add_callback(fence_excl, &dcb->cb, dma_buf_poll_cb)) { events &= ~pevents; + fence_put(fence_excl); } else { /* * No callback queued, wake up any additional * waiters. */ + fence_put(fence_excl); dma_buf_poll_cb(NULL, &dcb->cb); } } @@ -205,13 +219,25 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll) goto out;
for (i = 0; i < shared_count; ++i) { - struct fence *fence = fobj->shared[i]; - + struct fence *fence = fence_get_rcu(fobj->shared[i]); + if (!fence) { + /* + * fence refcount dropped to zero, this means + * that fobj has been freed + * + * call dma_buf_poll_cb and force a recheck! + */ + events &= ~POLLOUT; + dma_buf_poll_cb(NULL, &dcb->cb); + break; + } if (!fence_add_callback(fence, &dcb->cb, dma_buf_poll_cb)) { + fence_put(fence); events &= ~POLLOUT; break; } + fence_put(fence); }
/* No callback queued, wake up any additional waiters. */ @@ -220,7 +246,7 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll) }
out: - ww_mutex_unlock(&resv->lock); + rcu_read_unlock(); return events; }
diff --git a/drivers/base/reservation.c b/drivers/base/reservation.c index b82a5b630a8e..4cdce63140b8 100644 --- a/drivers/base/reservation.c +++ b/drivers/base/reservation.c @@ -87,9 +87,13 @@ reservation_object_add_shared_inplace(struct reservation_object *obj, struct fence *old_fence = fobj->shared[i];
fence_get(fence); + /* for the fence_get() */ + smp_mb__after_atomic_inc();
fobj->shared[i] = fence;
+ /* for the fence_put() */ + smp_mb__before_atomic_dec(); fence_put(old_fence); return; } @@ -141,8 +145,9 @@ reservation_object_add_shared_replace(struct reservation_object *obj, fobj->shared[fobj->shared_count++] = fence;
done: - obj->fence = fobj; - kfree(old); + rcu_assign_pointer(obj->fence, fobj); + if (old) + kfree_rcu(old, rcu); }
/* @@ -168,20 +173,25 @@ EXPORT_SYMBOL(reservation_object_add_shared_fence); void reservation_object_add_excl_fence(struct reservation_object *obj, struct fence *fence) { - struct fence *old_fence = obj->fence_excl; + struct fence *old_fence = reservation_object_get_excl(obj); struct reservation_object_list *old; u32 i = 0;
old = reservation_object_get_list(obj); - if (old) { + if (old) i = old->shared_count; - old->shared_count = 0; - }
if (fence) fence_get(fence);
- obj->fence_excl = fence; + rcu_assign_pointer(obj->fence_excl, fence); + + if (old) { + smp_wmb(); + old->shared_count = 0; + } + + smp_mb__before_atomic_dec();
/* inplace update, no shared fences */ while (i--) @@ -191,3 +201,126 @@ void reservation_object_add_excl_fence(struct reservation_object *obj, fence_put(old_fence); } EXPORT_SYMBOL(reservation_object_add_excl_fence); + +long reservation_object_wait_timeout_rcu(struct reservation_object *obj, + bool wait_all, bool intr, + unsigned long timeout) +{ + unsigned shared_count = 0; + + while (1) { + struct fence *fence = NULL, *fence_excl; + struct reservation_object_list *fobj; + long ret; + + rcu_read_lock(); + +retry: + if (wait_all) { + unsigned i; + + fobj = rcu_dereference(obj->fence); + if (fobj) + shared_count = ACCESS_ONCE(fobj->shared_count); + else + shared_count = 0; + smp_mb(); /* shared_count needs transitivity wrt fence_excl */ + + for (i = 0; i < shared_count; ++i) { + fence = ACCESS_ONCE(fobj->shared[i]); + + if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + continue; + + if (!fence_get_rcu(fence)) + goto retry; + + if (fence_is_signaled(fence)) + fence_put(fence); + else + break; + } + if (i == shared_count) + fence = NULL; + } + + if (shared_count == 0) { + fence_excl = rcu_dereference(obj->fence_excl); + if (fence_excl && !fence_get_rcu(fence_excl)) + goto retry; + + if (fence_excl && fence_is_signaled(fence_excl)) + fence_put(fence_excl); + else + fence = fence_excl; + } + + rcu_read_unlock(); + + if (fence == NULL) + return timeout; + + ret = fence_wait_timeout(fence, intr, timeout); + fence_put(fence); + + if (ret <= 0) + return ret; + else + timeout = ret; + } +} +EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu); + +bool reservation_object_test_signaled_rcu(struct reservation_object *obj, bool test_all) +{ + struct fence *fence, *fence_excl; + struct reservation_object_list *fobj; + unsigned shared_count = 0; + bool ret = true; + + rcu_read_lock(); + +retry: + if (test_all) { + unsigned i; + + fobj = rcu_dereference(obj->fence); + if (fobj) + shared_count = ACCESS_ONCE(fobj->shared_count); + else + shared_count = 0; + smp_mb(); /* shared_count needs transitivity wrt fence_excl */ + + for (i = 0; i < shared_count; ++i) { + fence = ACCESS_ONCE(fobj->shared[i]); + + if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + continue; + + if (!fence_get_rcu(fence)) + goto retry; + + ret = fence_is_signaled(fence); + fence_put(fence); + if (!ret) + break; + } + } + + if (shared_count == 0) { + fence_excl = rcu_dereference(obj->fence_excl); + + if (fence_excl && !test_bit(FENCE_FLAG_SIGNALED_BIT, &fence_excl->flags)) { + fence = fence_get_rcu(fence_excl); + if (!fence) + goto retry; + + ret = fence_is_signaled(fence); + fence_put(fence); + } + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu); diff --git a/include/linux/fence.h b/include/linux/fence.h index d13b5ab61726..8499ace045da 100644 --- a/include/linux/fence.h +++ b/include/linux/fence.h @@ -31,7 +31,7 @@ #include <linux/kref.h> #include <linux/sched.h> #include <linux/printk.h> -#include <linux/slab.h> +#include <linux/rcupdate.h>
struct fence; struct fence_ops; @@ -41,6 +41,7 @@ struct fence_cb; * struct fence - software synchronization primitive * @refcount: refcount for this fence * @ops: fence_ops associated with this fence + * @rcu: used for releasing fence with kfree_rcu * @cb_list: list of all callbacks to call * @lock: spin_lock_irqsave used for locking * @context: execution context this fence belongs to, returned by @@ -74,6 +75,7 @@ struct fence_cb; struct fence { struct kref refcount; const struct fence_ops *ops; + struct rcu_head rcu; struct list_head cb_list; spinlock_t *lock; unsigned context, seqno; @@ -190,11 +192,27 @@ static inline void fence_get(struct fence *fence) kref_get(&fence->refcount); }
+/** + * fence_get_rcu - get a fence from a reservation_object_list with rcu read lock + * @fence: [in] fence to increase refcount of + * + * Function returns NULL if no refcount could be obtained, or the fence. + */ +static inline struct fence *fence_get_rcu(struct fence *fence) +{ + struct fence *f = ACCESS_ONCE(fence); + + if (kref_get_unless_zero(&f->refcount)) + return f; + else + return NULL; +} + extern void release_fence(struct kref *kref);
static inline void free_fence(struct fence *fence) { - kfree(fence); + kfree_rcu(fence, rcu); }
/** diff --git a/include/linux/reservation.h b/include/linux/reservation.h index b602365c87f9..2a53bfbec6a1 100644 --- a/include/linux/reservation.h +++ b/include/linux/reservation.h @@ -42,10 +42,12 @@ #include <linux/ww_mutex.h> #include <linux/fence.h> #include <linux/slab.h> +#include <linux/rcupdate.h>
extern struct ww_class reservation_ww_class;
struct reservation_object_list { + struct rcu_head rcu; u32 shared_count, shared_max; struct fence *shared[]; }; @@ -53,21 +55,20 @@ struct reservation_object_list { struct reservation_object { struct ww_mutex lock;
- struct fence *fence_excl; - struct reservation_object_list *fence; + struct fence __rcu *fence_excl; + struct reservation_object_list __rcu *fence; struct reservation_object_list *staged; };
-#define reservation_object_assert_held(obj) \ - lockdep_assert_held(&(obj)->lock.base) +#define reservation_object_held(obj) (lockdep_is_held(&(obj)->lock.base))
static inline void reservation_object_init(struct reservation_object *obj) { ww_mutex_init(&obj->lock, &reservation_ww_class);
- obj->fence_excl = NULL; - obj->fence = NULL; + RCU_INIT_POINTER(obj->fence, NULL); + RCU_INIT_POINTER(obj->fence_excl, NULL); obj->staged = NULL; }
@@ -76,15 +77,17 @@ reservation_object_fini(struct reservation_object *obj) { int i; struct reservation_object_list *fobj; + struct fence *excl;
/* * This object should be dead and all references must have - * been released to it. + * been released to it, so no need to be protected with rcu. */ - if (obj->fence_excl) - fence_put(obj->fence_excl); + excl = rcu_dereference_protected(obj->fence_excl, 1); + if (excl) + fence_put(excl);
- fobj = obj->fence; + fobj = rcu_dereference_protected(obj->fence, 1); if (fobj) { for (i = 0; i < fobj->shared_count; ++i) fence_put(fobj->shared[i]); @@ -99,9 +102,15 @@ reservation_object_fini(struct reservation_object *obj) static inline struct reservation_object_list * reservation_object_get_list(struct reservation_object *obj) { - reservation_object_assert_held(obj); + return rcu_dereference_protected(obj->fence, + reservation_object_held(obj)); +}
- return obj->fence; +static inline struct fence * +reservation_object_get_excl(struct reservation_object *obj) +{ + return rcu_dereference_protected(obj->fence_excl, + reservation_object_held(obj)); }
int reservation_object_reserve_shared(struct reservation_object *obj); @@ -111,4 +120,11 @@ void reservation_object_add_shared_fence(struct reservation_object *obj, void reservation_object_add_excl_fence(struct reservation_object *obj, struct fence *fence);
+long reservation_object_wait_timeout_rcu(struct reservation_object *obj, + bool wait_all, bool intr, + unsigned long timeout); + +bool reservation_object_test_signaled_rcu(struct reservation_object *obj, + bool test_all); + #endif /* _LINUX_RESERVATION_H */