reservation: add suppport for read-only access using rcu (3c3b177a) · Commits · e / devices / android_kernel_fairphone_FP5

drivers/dma-buf/dma-buf.c

+36 −11

Original line number	Original line	Diff line number	Diff line
	@@ -137,7 +137,7 @@ static unsigned int dma_buf_poll(struct file file, poll_table poll)
	struct reservation_object_list *fobj;		struct reservation_object_list *fobj;
	struct fence *fence_excl;		struct fence *fence_excl;
	unsigned long events;		unsigned long events;
	unsigned shared_count;		unsigned shared_count, seq;

	dmabuf = file->private_data;		dmabuf = file->private_data;
	if (!dmabuf \|\| !dmabuf->resv)		if (!dmabuf \|\| !dmabuf->resv)
	@@ -151,14 +151,20 @@ static unsigned int dma_buf_poll(struct file file, poll_table poll)
	if (!events)		if (!events)
	return 0;		return 0;

	ww_mutex_lock(&resv->lock, NULL);		retry:
			seq = read_seqcount_begin(&resv->seq);
	fobj = resv->fence;		rcu_read_lock();
	if (!fobj)
	goto out;

			fobj = rcu_dereference(resv->fence);
			if (fobj)
	shared_count = fobj->shared_count;		shared_count = fobj->shared_count;
	fence_excl = resv->fence_excl;		else
			shared_count = 0;
			fence_excl = rcu_dereference(resv->fence_excl);
			if (read_seqcount_retry(&resv->seq, seq)) {
			rcu_read_unlock();
			goto retry;
			}

	if (fence_excl && (!(events & POLLOUT) \|\| shared_count == 0)) {		if (fence_excl && (!(events & POLLOUT) \|\| shared_count == 0)) {
	struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_excl;		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_excl;
	@@ -176,14 +182,20 @@ static unsigned int dma_buf_poll(struct file file, poll_table poll)
	spin_unlock_irq(&dmabuf->poll.lock);		spin_unlock_irq(&dmabuf->poll.lock);

	if (events & pevents) {		if (events & pevents) {
	if (!fence_add_callback(fence_excl, &dcb->cb,		if (!fence_get_rcu(fence_excl)) {
			/* force a recheck */
			events &= ~pevents;
			dma_buf_poll_cb(NULL, &dcb->cb);
			} else if (!fence_add_callback(fence_excl, &dcb->cb,
	dma_buf_poll_cb)) {		dma_buf_poll_cb)) {
	events &= ~pevents;		events &= ~pevents;
			fence_put(fence_excl);
	} else {		} else {
	/*		/*
	* No callback queued, wake up any additional		* No callback queued, wake up any additional
	* waiters.		* waiters.
	*/		*/
			fence_put(fence_excl);
	dma_buf_poll_cb(NULL, &dcb->cb);		dma_buf_poll_cb(NULL, &dcb->cb);
	}		}
	}		}
	@@ -205,13 +217,26 @@ static unsigned int dma_buf_poll(struct file file, poll_table poll)
	goto out;		goto out;

	for (i = 0; i < shared_count; ++i) {		for (i = 0; i < shared_count; ++i) {
	struct fence *fence = fobj->shared[i];		struct fence *fence = rcu_dereference(fobj->shared[i]);

			if (!fence_get_rcu(fence)) {
			/*
			* fence refcount dropped to zero, this means
			* that fobj has been freed
			*
			* call dma_buf_poll_cb and force a recheck!
			*/
			events &= ~POLLOUT;
			dma_buf_poll_cb(NULL, &dcb->cb);
			break;
			}
	if (!fence_add_callback(fence, &dcb->cb,		if (!fence_add_callback(fence, &dcb->cb,
	dma_buf_poll_cb)) {		dma_buf_poll_cb)) {
			fence_put(fence);
	events &= ~POLLOUT;		events &= ~POLLOUT;
	break;		break;
	}		}
			fence_put(fence);
	}		}

	/* No callback queued, wake up any additional waiters. */		/* No callback queued, wake up any additional waiters. */
	@@ -220,7 +245,7 @@ static unsigned int dma_buf_poll(struct file file, poll_table poll)
	}		}

	out:		out:
	ww_mutex_unlock(&resv->lock);		rcu_read_unlock();
	return events;		return events;
	}		}

drivers/dma-buf/fence.c

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -184,7 +184,7 @@ EXPORT_SYMBOL(fence_release);

	void fence_free(struct fence *fence)		void fence_free(struct fence *fence)
	{		{
	kfree(fence);		kfree_rcu(fence, rcu);
	}		}
	EXPORT_SYMBOL(fence_free);		EXPORT_SYMBOL(fence_free);

drivers/dma-buf/reservation.c

+310 −26

Original line number	Original line	Diff line number	Diff line
	@@ -38,6 +38,11 @@
	DEFINE_WW_CLASS(reservation_ww_class);		DEFINE_WW_CLASS(reservation_ww_class);
	EXPORT_SYMBOL(reservation_ww_class);		EXPORT_SYMBOL(reservation_ww_class);

			struct lock_class_key reservation_seqcount_class;
			EXPORT_SYMBOL(reservation_seqcount_class);

			const char reservation_seqcount_string[] = "reservation_seqcount";
			EXPORT_SYMBOL(reservation_seqcount_string);
	/*		/*
	* Reserve space to add a shared fence to a reservation_object,		* Reserve space to add a shared fence to a reservation_object,
	* must be called with obj->lock held.		* must be called with obj->lock held.
	@@ -82,27 +87,37 @@ reservation_object_add_shared_inplace(struct reservation_object *obj,
	{		{
	u32 i;		u32 i;

			fence_get(fence);

			preempt_disable();
			write_seqcount_begin(&obj->seq);

	for (i = 0; i < fobj->shared_count; ++i) {		for (i = 0; i < fobj->shared_count; ++i) {
	if (fobj->shared[i]->context == fence->context) {		struct fence *old_fence;
	struct fence *old_fence = fobj->shared[i];

	fence_get(fence);		old_fence = rcu_dereference_protected(fobj->shared[i],
			reservation_object_held(obj));

	fobj->shared[i] = fence;		if (old_fence->context == fence->context) {
			/* memory barrier is added by write_seqcount_begin */
			RCU_INIT_POINTER(fobj->shared[i], fence);
			write_seqcount_end(&obj->seq);
			preempt_enable();

	fence_put(old_fence);		fence_put(old_fence);
	return;		return;
	}		}
	}		}

	fence_get(fence);
	fobj->shared[fobj->shared_count] = fence;
	/*		/*
	* make the new fence visible before incrementing		* memory barrier is added by write_seqcount_begin,
	* fobj->shared_count		* fobj->shared_count is protected by this lock too
	*/		*/
	smp_wmb();		RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
	fobj->shared_count++;		fobj->shared_count++;

			write_seqcount_end(&obj->seq);
			preempt_enable();
	}		}

	static void		static void
	@@ -112,11 +127,12 @@ reservation_object_add_shared_replace(struct reservation_object *obj,
	struct fence *fence)		struct fence *fence)
	{		{
	unsigned i;		unsigned i;
			struct fence *old_fence = NULL;

	fence_get(fence);		fence_get(fence);

	if (!old) {		if (!old) {
	fobj->shared[0] = fence;		RCU_INIT_POINTER(fobj->shared[0], fence);
	fobj->shared_count = 1;		fobj->shared_count = 1;
	goto done;		goto done;
	}		}
	@@ -130,19 +146,38 @@ reservation_object_add_shared_replace(struct reservation_object *obj,
	fobj->shared_count = old->shared_count;		fobj->shared_count = old->shared_count;

	for (i = 0; i < old->shared_count; ++i) {		for (i = 0; i < old->shared_count; ++i) {
	if (fence && old->shared[i]->context == fence->context) {		struct fence *check;
	fence_put(old->shared[i]);
	fobj->shared[i] = fence;		check = rcu_dereference_protected(old->shared[i],
	fence = NULL;		reservation_object_held(obj));

			if (!old_fence && check->context == fence->context) {
			old_fence = check;
			RCU_INIT_POINTER(fobj->shared[i], fence);
	} else		} else
	fobj->shared[i] = old->shared[i];		RCU_INIT_POINTER(fobj->shared[i], check);
			}
			if (!old_fence) {
			RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
			fobj->shared_count++;
	}		}
	if (fence)
	fobj->shared[fobj->shared_count++] = fence;

	done:		done:
	obj->fence = fobj;		preempt_disable();
	kfree(old);		write_seqcount_begin(&obj->seq);
			/*
			* RCU_INIT_POINTER can be used here,
			* seqcount provides the necessary barriers
			*/
			RCU_INIT_POINTER(obj->fence, fobj);
			write_seqcount_end(&obj->seq);
			preempt_enable();

			if (old)
			kfree_rcu(old, rcu);

			if (old_fence)
			fence_put(old_fence);
	}		}

	/*		/*
	@@ -158,7 +193,7 @@ void reservation_object_add_shared_fence(struct reservation_object *obj,
	obj->staged = NULL;		obj->staged = NULL;

	if (!fobj) {		if (!fobj) {
	BUG_ON(old->shared_count == old->shared_max);		BUG_ON(old->shared_count >= old->shared_max);
	reservation_object_add_shared_inplace(obj, old, fence);		reservation_object_add_shared_inplace(obj, old, fence);
	} else		} else
	reservation_object_add_shared_replace(obj, old, fobj, fence);		reservation_object_add_shared_replace(obj, old, fobj, fence);
	@@ -168,26 +203,275 @@ EXPORT_SYMBOL(reservation_object_add_shared_fence);
	void reservation_object_add_excl_fence(struct reservation_object *obj,		void reservation_object_add_excl_fence(struct reservation_object *obj,
	struct fence *fence)		struct fence *fence)
	{		{
	struct fence *old_fence = obj->fence_excl;		struct fence *old_fence = reservation_object_get_excl(obj);
	struct reservation_object_list *old;		struct reservation_object_list *old;
	u32 i = 0;		u32 i = 0;

	old = reservation_object_get_list(obj);		old = reservation_object_get_list(obj);
	if (old) {		if (old)
	i = old->shared_count;		i = old->shared_count;
	old->shared_count = 0;
	}

	if (fence)		if (fence)
	fence_get(fence);		fence_get(fence);

	obj->fence_excl = fence;		preempt_disable();
			write_seqcount_begin(&obj->seq);
			/* write_seqcount_begin provides the necessary memory barrier */
			RCU_INIT_POINTER(obj->fence_excl, fence);
			if (old)
			old->shared_count = 0;
			write_seqcount_end(&obj->seq);
			preempt_enable();

	/* inplace update, no shared fences */		/* inplace update, no shared fences */
	while (i--)		while (i--)
	fence_put(old->shared[i]);		fence_put(rcu_dereference_protected(old->shared[i],
			reservation_object_held(obj)));

	if (old_fence)		if (old_fence)
	fence_put(old_fence);		fence_put(old_fence);
	}		}
	EXPORT_SYMBOL(reservation_object_add_excl_fence);		EXPORT_SYMBOL(reservation_object_add_excl_fence);

			int reservation_object_get_fences_rcu(struct reservation_object *obj,
			struct fence **pfence_excl,
			unsigned *pshared_count,
			struct fence ***pshared)
			{
			unsigned shared_count = 0;
			unsigned retry = 1;
			struct fence *shared = NULL, fence_excl = NULL;
			int ret = 0;

			while (retry) {
			struct reservation_object_list *fobj;
			unsigned seq;

			seq = read_seqcount_begin(&obj->seq);

			rcu_read_lock();

			fobj = rcu_dereference(obj->fence);
			if (fobj) {
			struct fence **nshared;
			size_t sz = sizeof(shared) fobj->shared_max;

			nshared = krealloc(shared, sz,
			GFP_NOWAIT \| __GFP_NOWARN);
			if (!nshared) {
			rcu_read_unlock();
			nshared = krealloc(shared, sz, GFP_KERNEL);
			if (nshared) {
			shared = nshared;
			continue;
			}

			ret = -ENOMEM;
			shared_count = 0;
			break;
			}
			shared = nshared;
			memcpy(shared, fobj->shared, sz);
			shared_count = fobj->shared_count;
			} else
			shared_count = 0;
			fence_excl = rcu_dereference(obj->fence_excl);

			retry = read_seqcount_retry(&obj->seq, seq);
			if (retry)
			goto unlock;

			if (!fence_excl \|\| fence_get_rcu(fence_excl)) {
			unsigned i;

			for (i = 0; i < shared_count; ++i) {
			if (fence_get_rcu(shared[i]))
			continue;

			/* uh oh, refcount failed, abort and retry */
			while (i--)
			fence_put(shared[i]);

			if (fence_excl) {
			fence_put(fence_excl);
			fence_excl = NULL;
			}

			retry = 1;
			break;
			}
			} else
			retry = 1;

			unlock:
			rcu_read_unlock();
			}
			*pshared_count = shared_count;
			if (shared_count)
			*pshared = shared;
			else {
			*pshared = NULL;
			kfree(shared);
			}
			*pfence_excl = fence_excl;

			return ret;
			}
			EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu);

			long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
			bool wait_all, bool intr,
			unsigned long timeout)
			{
			struct fence *fence;
			unsigned seq, shared_count, i = 0;
			long ret = timeout;

			retry:
			fence = NULL;
			shared_count = 0;
			seq = read_seqcount_begin(&obj->seq);
			rcu_read_lock();

			if (wait_all) {
			struct reservation_object_list *fobj = rcu_dereference(obj->fence);

			if (fobj)
			shared_count = fobj->shared_count;

			if (read_seqcount_retry(&obj->seq, seq))
			goto unlock_retry;

			for (i = 0; i < shared_count; ++i) {
			struct fence *lfence = rcu_dereference(fobj->shared[i]);

			if (test_bit(FENCE_FLAG_SIGNALED_BIT, &lfence->flags))
			continue;

			if (!fence_get_rcu(lfence))
			goto unlock_retry;

			if (fence_is_signaled(lfence)) {
			fence_put(lfence);
			continue;
			}

			fence = lfence;
			break;
			}
			}

			if (!shared_count) {
			struct fence *fence_excl = rcu_dereference(obj->fence_excl);

			if (read_seqcount_retry(&obj->seq, seq))
			goto unlock_retry;

			if (fence_excl &&
			!test_bit(FENCE_FLAG_SIGNALED_BIT, &fence_excl->flags)) {
			if (!fence_get_rcu(fence_excl))
			goto unlock_retry;

			if (fence_is_signaled(fence_excl))
			fence_put(fence_excl);
			else
			fence = fence_excl;
			}
			}

			rcu_read_unlock();
			if (fence) {
			ret = fence_wait_timeout(fence, intr, ret);
			fence_put(fence);
			if (ret > 0 && wait_all && (i + 1 < shared_count))
			goto retry;
			}
			return ret;

			unlock_retry:
			rcu_read_unlock();
			goto retry;
			}
			EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu);


			static inline int
			reservation_object_test_signaled_single(struct fence *passed_fence)
			{
			struct fence fence, lfence = passed_fence;
			int ret = 1;

			if (!test_bit(FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) {
			int ret;

			fence = fence_get_rcu(lfence);
			if (!fence)
			return -1;

			ret = !!fence_is_signaled(fence);
			fence_put(fence);
			}
			return ret;
			}

			bool reservation_object_test_signaled_rcu(struct reservation_object *obj,
			bool test_all)
			{
			unsigned seq, shared_count;
			int ret = true;

			retry:
			shared_count = 0;
			seq = read_seqcount_begin(&obj->seq);
			rcu_read_lock();

			if (test_all) {
			unsigned i;

			struct reservation_object_list *fobj = rcu_dereference(obj->fence);

			if (fobj)
			shared_count = fobj->shared_count;

			if (read_seqcount_retry(&obj->seq, seq))
			goto unlock_retry;

			for (i = 0; i < shared_count; ++i) {
			struct fence *fence = rcu_dereference(fobj->shared[i]);

			ret = reservation_object_test_signaled_single(fence);
			if (ret < 0)
			goto unlock_retry;
			else if (!ret)
			break;
			}

			/*
			* There could be a read_seqcount_retry here, but nothing cares
			* about whether it's the old or newer fence pointers that are
			* signaled. That race could still have happened after checking
			* read_seqcount_retry. If you care, use ww_mutex_lock.
			*/
			}

			if (!shared_count) {
			struct fence *fence_excl = rcu_dereference(obj->fence_excl);

			if (read_seqcount_retry(&obj->seq, seq))
			goto unlock_retry;

			if (fence_excl) {
			ret = reservation_object_test_signaled_single(fence_excl);
			if (ret < 0)
			goto unlock_retry;
			}
			}

			rcu_read_unlock();
			return ret;

			unlock_retry:
			rcu_read_unlock();
			goto retry;
			}
			EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu);

include/linux/fence.h

+17 −0

Original line number	Original line	Diff line number	Diff line
	@@ -28,6 +28,7 @@
	#include <linux/kref.h>		#include <linux/kref.h>
	#include <linux/sched.h>		#include <linux/sched.h>
	#include <linux/printk.h>		#include <linux/printk.h>
			#include <linux/rcupdate.h>

	struct fence;		struct fence;
	struct fence_ops;		struct fence_ops;
	@@ -37,6 +38,7 @@ struct fence_cb;
	* struct fence - software synchronization primitive		* struct fence - software synchronization primitive
	* @refcount: refcount for this fence		* @refcount: refcount for this fence
	* @ops: fence_ops associated with this fence		* @ops: fence_ops associated with this fence
			* @rcu: used for releasing fence with kfree_rcu
	* @cb_list: list of all callbacks to call		* @cb_list: list of all callbacks to call
	* @lock: spin_lock_irqsave used for locking		* @lock: spin_lock_irqsave used for locking
	* @context: execution context this fence belongs to, returned by		* @context: execution context this fence belongs to, returned by
	@@ -70,6 +72,7 @@ struct fence_cb;
	struct fence {		struct fence {
	struct kref refcount;		struct kref refcount;
	const struct fence_ops *ops;		const struct fence_ops *ops;
			struct rcu_head rcu;
	struct list_head cb_list;		struct list_head cb_list;
	spinlock_t *lock;		spinlock_t *lock;
	unsigned context, seqno;		unsigned context, seqno;
	@@ -191,6 +194,20 @@ static inline struct fence fence_get(struct fence fence)
	return fence;		return fence;
	}		}

			/**
			* fence_get_rcu - get a fence from a reservation_object_list with rcu read lock
			* @fence: [in] fence to increase refcount of
			*
			* Function returns NULL if no refcount could be obtained, or the fence.
			*/
			static inline struct fence fence_get_rcu(struct fence fence)
			{
			if (kref_get_unless_zero(&fence->refcount))
			return fence;
			else
			return NULL;
			}

	/**		/**
	* fence_put - decreases refcount of the fence		* fence_put - decreases refcount of the fence
	* @fence: [in] fence to reduce refcount of		* @fence: [in] fence to reduce refcount of

include/linux/reservation.h

+36 −16

Original line number	Original line	Diff line number	Diff line
	@@ -42,22 +42,29 @@
	#include <linux/ww_mutex.h>		#include <linux/ww_mutex.h>
	#include <linux/fence.h>		#include <linux/fence.h>
	#include <linux/slab.h>		#include <linux/slab.h>
			#include <linux/seqlock.h>
			#include <linux/rcupdate.h>

	extern struct ww_class reservation_ww_class;		extern struct ww_class reservation_ww_class;
			extern struct lock_class_key reservation_seqcount_class;
			extern const char reservation_seqcount_string[];

	struct reservation_object_list {		struct reservation_object_list {
			struct rcu_head rcu;
	u32 shared_count, shared_max;		u32 shared_count, shared_max;
	struct fence *shared[];		struct fence __rcu *shared[];
	};		};

	struct reservation_object {		struct reservation_object {
	struct ww_mutex lock;		struct ww_mutex lock;
			seqcount_t seq;

	struct fence *fence_excl;		struct fence __rcu *fence_excl;
	struct reservation_object_list *fence;		struct reservation_object_list __rcu *fence;
	struct reservation_object_list *staged;		struct reservation_object_list *staged;
	};		};

			#define reservation_object_held(obj) lockdep_is_held(&(obj)->lock.base)
	#define reservation_object_assert_held(obj) \		#define reservation_object_assert_held(obj) \
	lockdep_assert_held(&(obj)->lock.base)		lockdep_assert_held(&(obj)->lock.base)

	@@ -66,8 +73,9 @@ reservation_object_init(struct reservation_object *obj)
	{		{
	ww_mutex_init(&obj->lock, &reservation_ww_class);		ww_mutex_init(&obj->lock, &reservation_ww_class);

	obj->fence_excl = NULL;		__seqcount_init(&obj->seq, reservation_seqcount_string, &reservation_seqcount_class);
	obj->fence = NULL;		RCU_INIT_POINTER(obj->fence, NULL);
			RCU_INIT_POINTER(obj->fence_excl, NULL);
	obj->staged = NULL;		obj->staged = NULL;
	}		}

	@@ -76,18 +84,20 @@ reservation_object_fini(struct reservation_object *obj)
	{		{
	int i;		int i;
	struct reservation_object_list *fobj;		struct reservation_object_list *fobj;
			struct fence *excl;

	/*		/*
	* This object should be dead and all references must have		* This object should be dead and all references must have
	* been released to it.		* been released to it, so no need to be protected with rcu.
	*/		*/
	if (obj->fence_excl)		excl = rcu_dereference_protected(obj->fence_excl, 1);
	fence_put(obj->fence_excl);		if (excl)
			fence_put(excl);

	fobj = obj->fence;		fobj = rcu_dereference_protected(obj->fence, 1);
	if (fobj) {		if (fobj) {
	for (i = 0; i < fobj->shared_count; ++i)		for (i = 0; i < fobj->shared_count; ++i)
	fence_put(fobj->shared[i]);		fence_put(rcu_dereference_protected(fobj->shared[i], 1));

	kfree(fobj);		kfree(fobj);
	}		}
	@@ -99,17 +109,15 @@ reservation_object_fini(struct reservation_object *obj)
	static inline struct reservation_object_list *		static inline struct reservation_object_list *
	reservation_object_get_list(struct reservation_object *obj)		reservation_object_get_list(struct reservation_object *obj)
	{		{
	reservation_object_assert_held(obj);		return rcu_dereference_protected(obj->fence,
			reservation_object_held(obj));
	return obj->fence;
	}		}

	static inline struct fence *		static inline struct fence *
	reservation_object_get_excl(struct reservation_object *obj)		reservation_object_get_excl(struct reservation_object *obj)
	{		{
	reservation_object_assert_held(obj);		return rcu_dereference_protected(obj->fence_excl,
			reservation_object_held(obj));
	return obj->fence_excl;
	}		}

	int reservation_object_reserve_shared(struct reservation_object *obj);		int reservation_object_reserve_shared(struct reservation_object *obj);
	@@ -119,4 +127,16 @@ void reservation_object_add_shared_fence(struct reservation_object *obj,
	void reservation_object_add_excl_fence(struct reservation_object *obj,		void reservation_object_add_excl_fence(struct reservation_object *obj,
	struct fence *fence);		struct fence *fence);

			int reservation_object_get_fences_rcu(struct reservation_object *obj,
			struct fence **pfence_excl,
			unsigned *pshared_count,
			struct fence ***pshared);

			long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
			bool wait_all, bool intr,
			unsigned long timeout);

			bool reservation_object_test_signaled_rcu(struct reservation_object *obj,
			bool test_all);

	#endif /* _LINUX_RESERVATION_H */		#endif /* _LINUX_RESERVATION_H */