Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6baa5c60 authored by Nicolai Hähnle's avatar Nicolai Hähnle Committed by Ingo Molnar
Browse files

locking/ww_mutex: Add waiters in stamp order 



Add regular waiters in stamp order. Keep adding waiters that have no
context in FIFO order and take care not to starve them.

While adding our task as a waiter, back off if we detect that there is
a waiter with a lower stamp in front of us.

Make sure to call lock_contended even when we back off early.

For w/w mutexes, being first in the wait list is only stable when
taking the lock without a context. Therefore, the purpose of the first
flag is split into two: 'first' remains to indicate whether we want to
spin optimistically, while 'handoff' indicates that we should be
prepared to accept a handoff.

For w/w locking with a context, we always accept handoffs after the
first schedule(), to handle the following sequence of events:

 1. Task #0 unlocks and hands off to Task #2 which is first in line

 2. Task #1 adds itself in front of Task #2

 3. Task #2 wakes up and must accept the handoff even though it is no
    longer first in line

Signed-off-by: default avatarNicolai Hähnle <nicolai.haehnle@amd.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= <Nicolai.Haehnle@amd.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Daniel Vetter <daniel@ffwll.ch>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Maarten Lankhorst <dev@mblankhorst.nl>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: dri-devel@lists.freedesktop.org
Link: http://lkml.kernel.org/r/1482346000-9927-7-git-send-email-nhaehnle@gmail.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent c5470b22

include/linux/mutex.h

+3 −0
Original line number Diff line number Diff line
@@ -20,6 +20,8 @@ 
#include <linux/osq_lock.h>

#include <linux/debug_locks.h>



struct ww_acquire_ctx;



/*

 * Simple, straightforward mutexes with strict semantics:

 *
@@ -75,6 +77,7 @@ static inline struct task_struct *__mutex_owner(struct mutex *lock) 
struct mutex_waiter {

	struct list_head	list;

	struct task_struct	*task;

	struct ww_acquire_ctx	*ww_ctx;

#ifdef CONFIG_DEBUG_MUTEXES

	void			*magic;

#endif

kernel/locking/mutex.c

+69 −7
Original line number Diff line number Diff line
@@ -615,6 +615,52 @@ __ww_mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx) 
	return 0;

}



static inline int __sched

__ww_mutex_add_waiter(struct mutex_waiter *waiter,

		      struct mutex *lock,

		      struct ww_acquire_ctx *ww_ctx)

{

	struct mutex_waiter *cur;

	struct list_head *pos;



	if (!ww_ctx) {

		list_add_tail(&waiter->list, &lock->wait_list);

		return 0;

	}



	/*

	 * Add the waiter before the first waiter with a higher stamp.

	 * Waiters without a context are skipped to avoid starving

	 * them.

	 */

	pos = &lock->wait_list;

	list_for_each_entry_reverse(cur, &lock->wait_list, list) {

		if (!cur->ww_ctx)

			continue;



		if (__ww_ctx_stamp_after(ww_ctx, cur->ww_ctx)) {

			/* Back off immediately if necessary. */

			if (ww_ctx->acquired > 0) {

#ifdef CONFIG_DEBUG_MUTEXES

				struct ww_mutex *ww;



				ww = container_of(lock, struct ww_mutex, base);

				DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock);

				ww_ctx->contending_lock = ww;

#endif

				return -EDEADLK;

			}



			break;

		}



		pos = &cur->list;

	}



	list_add_tail(&waiter->list, pos);

	return 0;

}



/*

 * Lock a mutex (possibly interruptible), slowpath:

 */
@@ -659,15 +705,25 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, 
	debug_mutex_lock_common(lock, &waiter);

	debug_mutex_add_waiter(lock, &waiter, current);



	lock_contended(&lock->dep_map, ip);



	if (!use_ww_ctx) {

		/* add waiting tasks to the end of the waitqueue (FIFO): */

		list_add_tail(&waiter.list, &lock->wait_list);

	} else {

		/* Add in stamp order, waking up waiters that must back off. */

		ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx);

		if (ret)

			goto err_early_backoff;



		waiter.ww_ctx = ww_ctx;

	}



	waiter.task = current;



	if (__mutex_waiter_is_first(lock, &waiter))

		__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);



	lock_contended(&lock->dep_map, ip);



	set_current_state(state);

	for (;;) {

		/*
@@ -698,8 +754,13 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, 
		spin_unlock_mutex(&lock->wait_lock, flags);

		schedule_preempt_disabled();



		if (!first && __mutex_waiter_is_first(lock, &waiter)) {

			first = true;

		/*

		 * ww_mutex needs to always recheck its position since its waiter

		 * list is not FIFO ordered.

		 */

		if ((use_ww_ctx && ww_ctx) || !first) {

			first = __mutex_waiter_is_first(lock, &waiter);

			if (first)

				__mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);

		}
@@ -739,6 +800,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, 
err:

	__set_current_state(TASK_RUNNING);

	mutex_remove_waiter(lock, &waiter, current);

err_early_backoff:

	spin_unlock_mutex(&lock->wait_lock, flags);

	debug_mutex_free_waiter(&waiter);

	mutex_release(&lock->dep_map, 1, ip);