locking/mutexes: Refactor optimistic spinning code (76916515) · Commits · e / devices / android_kernel_fairphone_FP4

kernel/locking/mutex.c

+214 −182

Original line number	Diff line number	Diff line
		@@ -106,6 +106,92 @@ void __sched mutex_lock(struct mutex *lock)
		EXPORT_SYMBOL(mutex_lock);
		#endif

		static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
		struct ww_acquire_ctx *ww_ctx)
		{
		#ifdef CONFIG_DEBUG_MUTEXES
		/*
		* If this WARN_ON triggers, you used ww_mutex_lock to acquire,
		* but released with a normal mutex_unlock in this call.
		*
		* This should never happen, always use ww_mutex_unlock.
		*/
		DEBUG_LOCKS_WARN_ON(ww->ctx);

		/*
		* Not quite done after calling ww_acquire_done() ?
		*/
		DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);

		if (ww_ctx->contending_lock) {
		/*
		* After -EDEADLK you tried to
		* acquire a different ww_mutex? Bad!
		*/
		DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);

		/*
		* You called ww_mutex_lock after receiving -EDEADLK,
		* but 'forgot' to unlock everything else first?
		*/
		DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
		ww_ctx->contending_lock = NULL;
		}

		/*
		* Naughty, using a different class will lead to undefined behavior!
		*/
		DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
		#endif
		ww_ctx->acquired++;
		}

		/*
		* after acquiring lock with fastpath or when we lost out in contested
		* slowpath, set ctx and wake up any waiters so they can recheck.
		*
		* This function is never called when CONFIG_DEBUG_LOCK_ALLOC is set,
		* as the fastpath and opportunistic spinning are disabled in that case.
		*/
		static __always_inline void
		ww_mutex_set_context_fastpath(struct ww_mutex *lock,
		struct ww_acquire_ctx *ctx)
		{
		unsigned long flags;
		struct mutex_waiter *cur;

		ww_mutex_lock_acquired(lock, ctx);

		lock->ctx = ctx;

		/*
		* The lock->ctx update should be visible on all cores before
		* the atomic read is done, otherwise contended waiters might be
		* missed. The contended waiters will either see ww_ctx == NULL
		* and keep spinning, or it will acquire wait_lock, add itself
		* to waiter list and sleep.
		*/
		smp_mb(); /* ^^^ */

		/*
		* Check if lock is contended, if not there is nobody to wake up
		*/
		if (likely(atomic_read(&lock->base.count) == 0))
		return;

		/*
		* Uh oh, we raced in fastpath, wake up everyone in this case,
		* so they can see the new lock->ctx.
		*/
		spin_lock_mutex(&lock->base.wait_lock, flags);
		list_for_each_entry(cur, &lock->base.wait_list, list) {
		debug_mutex_wake_waiter(&lock->base, cur);
		wake_up_process(cur->task);
		}
		spin_unlock_mutex(&lock->base.wait_lock, flags);
		}


		#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
		/*
		* In order to avoid a stampede of mutex spinners from acquiring the mutex
		@@ -180,6 +266,129 @@ static inline int mutex_can_spin_on_owner(struct mutex *lock)
		*/
		return retval;
		}

		/*
		* Atomically try to take the lock when it is available
		*/
		static inline bool mutex_try_to_acquire(struct mutex *lock)
		{
		return !mutex_is_locked(lock) &&
		(atomic_cmpxchg(&lock->count, 1, 0) == 1);
		}

		/*
		* Optimistic spinning.
		*
		* We try to spin for acquisition when we find that the lock owner
		* is currently running on a (different) CPU and while we don't
		* need to reschedule. The rationale is that if the lock owner is
		* running, it is likely to release the lock soon.
		*
		* Since this needs the lock owner, and this mutex implementation
		* doesn't track the owner atomically in the lock field, we need to
		* track it non-atomically.
		*
		* We can't do this for DEBUG_MUTEXES because that relies on wait_lock
		* to serialize everything.
		*
		* The mutex spinners are queued up using MCS lock so that only one
		* spinner can compete for the mutex. However, if mutex spinning isn't
		* going to happen, there is no point in going through the lock/unlock
		* overhead.
		*
		* Returns true when the lock was taken, otherwise false, indicating
		* that we need to jump to the slowpath and sleep.
		*/
		static bool mutex_optimistic_spin(struct mutex *lock,
		struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
		{
		struct task_struct *task = current;

		if (!mutex_can_spin_on_owner(lock))
		goto done;

		if (!osq_lock(&lock->osq))
		goto done;

		while (true) {
		struct task_struct *owner;

		if (use_ww_ctx && ww_ctx->acquired > 0) {
		struct ww_mutex *ww;

		ww = container_of(lock, struct ww_mutex, base);
		/*
		* If ww->ctx is set the contents are undefined, only
		* by acquiring wait_lock there is a guarantee that
		* they are not invalid when reading.
		*
		* As such, when deadlock detection needs to be
		* performed the optimistic spinning cannot be done.
		*/
		if (ACCESS_ONCE(ww->ctx))
		break;
		}

		/*
		* If there's an owner, wait for it to either
		* release the lock or go to sleep.
		*/
		owner = ACCESS_ONCE(lock->owner);
		if (owner && !mutex_spin_on_owner(lock, owner))
		break;

		/* Try to acquire the mutex if it is unlocked. */
		if (mutex_try_to_acquire(lock)) {
		lock_acquired(&lock->dep_map, ip);

		if (use_ww_ctx) {
		struct ww_mutex *ww;
		ww = container_of(lock, struct ww_mutex, base);

		ww_mutex_set_context_fastpath(ww, ww_ctx);
		}

		mutex_set_owner(lock);
		osq_unlock(&lock->osq);
		return true;
		}

		/*
		* When there's no owner, we might have preempted between the
		* owner acquiring the lock and setting the owner field. If
		* we're an RT task that will live-lock because we won't let
		* the owner complete.
		*/
		if (!owner && (need_resched() \|\| rt_task(task)))
		break;

		/*
		* The cpu_relax() call is a compiler barrier which forces
		* everything in this loop to be re-loaded. We don't need
		* memory barriers as we'll eventually observe the right
		* values at the cost of a few extra spins.
		*/
		cpu_relax_lowlatency();
		}

		osq_unlock(&lock->osq);
		done:
		/*
		* If we fell out of the spin path because of need_resched(),
		* reschedule now, before we try-lock the mutex. This avoids getting
		* scheduled out right after we obtained the mutex.
		*/
		if (need_resched())
		schedule_preempt_disabled();

		return false;
		}
		#else
		static bool mutex_optimistic_spin(struct mutex *lock,
		struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
		{
		return false;
		}
		#endif

		__visible __used noinline
		@@ -277,91 +486,6 @@ __mutex_lock_check_stamp(struct mutex lock, struct ww_acquire_ctx ctx)
		return 0;
		}

		static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
		struct ww_acquire_ctx *ww_ctx)
		{
		#ifdef CONFIG_DEBUG_MUTEXES
		/*
		* If this WARN_ON triggers, you used ww_mutex_lock to acquire,
		* but released with a normal mutex_unlock in this call.
		*
		* This should never happen, always use ww_mutex_unlock.
		*/
		DEBUG_LOCKS_WARN_ON(ww->ctx);

		/*
		* Not quite done after calling ww_acquire_done() ?
		*/
		DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);

		if (ww_ctx->contending_lock) {
		/*
		* After -EDEADLK you tried to
		* acquire a different ww_mutex? Bad!
		*/
		DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);

		/*
		* You called ww_mutex_lock after receiving -EDEADLK,
		* but 'forgot' to unlock everything else first?
		*/
		DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
		ww_ctx->contending_lock = NULL;
		}

		/*
		* Naughty, using a different class will lead to undefined behavior!
		*/
		DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
		#endif
		ww_ctx->acquired++;
		}

		/*
		* after acquiring lock with fastpath or when we lost out in contested
		* slowpath, set ctx and wake up any waiters so they can recheck.
		*
		* This function is never called when CONFIG_DEBUG_LOCK_ALLOC is set,
		* as the fastpath and opportunistic spinning are disabled in that case.
		*/
		static __always_inline void
		ww_mutex_set_context_fastpath(struct ww_mutex *lock,
		struct ww_acquire_ctx *ctx)
		{
		unsigned long flags;
		struct mutex_waiter *cur;

		ww_mutex_lock_acquired(lock, ctx);

		lock->ctx = ctx;

		/*
		* The lock->ctx update should be visible on all cores before
		* the atomic read is done, otherwise contended waiters might be
		* missed. The contended waiters will either see ww_ctx == NULL
		* and keep spinning, or it will acquire wait_lock, add itself
		* to waiter list and sleep.
		*/
		smp_mb(); /* ^^^ */

		/*
		* Check if lock is contended, if not there is nobody to wake up
		*/
		if (likely(atomic_read(&lock->base.count) == 0))
		return;

		/*
		* Uh oh, we raced in fastpath, wake up everyone in this case,
		* so they can see the new lock->ctx.
		*/
		spin_lock_mutex(&lock->base.wait_lock, flags);
		list_for_each_entry(cur, &lock->base.wait_list, list) {
		debug_mutex_wake_waiter(&lock->base, cur);
		wake_up_process(cur->task);
		}
		spin_unlock_mutex(&lock->base.wait_lock, flags);
		}

		/*
		* Lock a mutex (possibly interruptible), slowpath:
		*/
		@@ -378,104 +502,12 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
		preempt_disable();
		mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);

		#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
		/*
		* Optimistic spinning.
		*
		* We try to spin for acquisition when we find that the lock owner
		* is currently running on a (different) CPU and while we don't
		* need to reschedule. The rationale is that if the lock owner is
		* running, it is likely to release the lock soon.
		*
		* Since this needs the lock owner, and this mutex implementation
		* doesn't track the owner atomically in the lock field, we need to
		* track it non-atomically.
		*
		* We can't do this for DEBUG_MUTEXES because that relies on wait_lock
		* to serialize everything.
		*
		* The mutex spinners are queued up using MCS lock so that only one
		* spinner can compete for the mutex. However, if mutex spinning isn't
		* going to happen, there is no point in going through the lock/unlock
		* overhead.
		*/
		if (!mutex_can_spin_on_owner(lock))
		goto slowpath;

		if (!osq_lock(&lock->osq))
		goto slowpath;

		for (;;) {
		struct task_struct *owner;

		if (use_ww_ctx && ww_ctx->acquired > 0) {
		struct ww_mutex *ww;

		ww = container_of(lock, struct ww_mutex, base);
		/*
		* If ww->ctx is set the contents are undefined, only
		* by acquiring wait_lock there is a guarantee that
		* they are not invalid when reading.
		*
		* As such, when deadlock detection needs to be
		* performed the optimistic spinning cannot be done.
		*/
		if (ACCESS_ONCE(ww->ctx))
		break;
		}

		/*
		* If there's an owner, wait for it to either
		* release the lock or go to sleep.
		*/
		owner = ACCESS_ONCE(lock->owner);
		if (owner && !mutex_spin_on_owner(lock, owner))
		break;

		/* Try to acquire the mutex if it is unlocked. */
		if (!mutex_is_locked(lock) &&
		(atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
		lock_acquired(&lock->dep_map, ip);
		if (use_ww_ctx) {
		struct ww_mutex *ww;
		ww = container_of(lock, struct ww_mutex, base);

		ww_mutex_set_context_fastpath(ww, ww_ctx);
		}

		mutex_set_owner(lock);
		osq_unlock(&lock->osq);
		if (mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) {
		/* got the lock, yay! */
		preempt_enable();
		return 0;
		}

		/*
		* When there's no owner, we might have preempted between the
		* owner acquiring the lock and setting the owner field. If
		* we're an RT task that will live-lock because we won't let
		* the owner complete.
		*/
		if (!owner && (need_resched() \|\| rt_task(task)))
		break;

		/*
		* The cpu_relax() call is a compiler barrier which forces
		* everything in this loop to be re-loaded. We don't need
		* memory barriers as we'll eventually observe the right
		* values at the cost of a few extra spins.
		*/
		cpu_relax_lowlatency();
		}
		osq_unlock(&lock->osq);
		slowpath:
		/*
		* If we fell out of the spin path because of need_resched(),
		* reschedule now, before we try-lock the mutex. This avoids getting
		* scheduled out right after we obtained the mutex.
		*/
		if (need_resched())
		schedule_preempt_disabled();
		#endif
		spin_lock_mutex(&lock->wait_lock, flags);

		/*