Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ca5f9524 authored by Darren Hart's avatar Darren Hart Committed by Thomas Gleixner
Browse files

futex: separate futex_wait_queue_me() logic from futex_wait() 



Refactor futex_wait() in preparation for futex_wait_requeue_pi(). In
order to reuse a good chunk of the futex_wait() code for the upcoming
futex_wait_requeue_pi() function, this patch breaks out the
queue-to-wakeup section of futex_wait() into futex_wait_queue_me().

Signed-off-by: default avatarDarren Hart <dvhltc@us.ibm.com>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent 6bb59750

kernel/futex.c

+76 −62
Original line number Diff line number Diff line
@@ -1115,24 +1115,87 @@ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, 


static long futex_wait_restart(struct restart_block *restart);



/**

 * futex_wait_queue_me() - queue_me() and wait for wakeup, timeout, or signal

 * @hb:		the futex hash bucket, must be locked by the caller

 * @q:		the futex_q to queue up on

 * @timeout:	the prepared hrtimer_sleeper, or null for no timeout

 * @wait:	the wait_queue to add to the futex_q after queueing in the hb

 */

static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,

				struct hrtimer_sleeper *timeout,

				wait_queue_t *wait)

{

	queue_me(q, hb);



	/*

	 * There might have been scheduling since the queue_me(), as we

	 * cannot hold a spinlock across the get_user() in case it

	 * faults, and we cannot just set TASK_INTERRUPTIBLE state when

	 * queueing ourselves into the futex hash.  This code thus has to

	 * rely on the futex_wake() code removing us from hash when it

	 * wakes us up.

	 */



	/* add_wait_queue is the barrier after __set_current_state. */

	__set_current_state(TASK_INTERRUPTIBLE);



	/*

	 * Add current as the futex_q waiter.  We don't remove ourselves from

	 * the wait_queue because we are the only user of it.

	 */

	add_wait_queue(&q->waiter, wait);



	/* Arm the timer */

	if (timeout) {

		hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);

		if (!hrtimer_active(&timeout->timer))

			timeout->task = NULL;

	}



	/*

	 * !plist_node_empty() is safe here without any lock.

	 * q.lock_ptr != 0 is not safe, because of ordering against wakeup.

	 */

	if (likely(!plist_node_empty(&q->list))) {

		/*

		 * If the timer has already expired, current will already be

		 * flagged for rescheduling. Only call schedule if there

		 * is no timeout, or if it has yet to expire.

		 */

		if (!timeout || timeout->task)

			schedule();

	}

	__set_current_state(TASK_RUNNING);

}



static int futex_wait(u32 __user *uaddr, int fshared,

		      u32 val, ktime_t *abs_time, u32 bitset, int clockrt)

{

	struct task_struct *curr = current;

	struct hrtimer_sleeper timeout, *to = NULL;

	DECLARE_WAITQUEUE(wait, current);

	struct restart_block *restart;

	DECLARE_WAITQUEUE(wait, curr);

	struct futex_hash_bucket *hb;

	struct futex_q q;

	u32 uval;

	int ret;

	struct hrtimer_sleeper t;

	int rem = 0;



	if (!bitset)

		return -EINVAL;



	q.pi_state = NULL;

	q.bitset = bitset;



	if (abs_time) {

		to = &timeout;



		hrtimer_init_on_stack(&to->timer, clockrt ? CLOCK_REALTIME :

				      CLOCK_MONOTONIC, HRTIMER_MODE_ABS);

		hrtimer_init_sleeper(to, current);

		hrtimer_set_expires_range_ns(&to->timer, *abs_time,

					     current->timer_slack_ns);

	}



retry:

	q.key = FUTEX_KEY_INIT;

	ret = get_futex_key(uaddr, fshared, &q.key);
@@ -1178,75 +1241,22 @@ static int futex_wait(u32 __user *uaddr, int fshared, 
		goto retry;

	}

	ret = -EWOULDBLOCK;



	/* Only actually queue if *uaddr contained val.  */

	if (unlikely(uval != val)) {

		queue_unlock(&q, hb);

		goto out_put_key;

	}



	/* Only actually queue if *uaddr contained val.  */

	queue_me(&q, hb);



	/*

	 * There might have been scheduling since the queue_me(), as we

	 * cannot hold a spinlock across the get_user() in case it

	 * faults, and we cannot just set TASK_INTERRUPTIBLE state when

	 * queueing ourselves into the futex hash.  This code thus has to

	 * rely on the futex_wake() code removing us from hash when it

	 * wakes us up.

	 */



	/* add_wait_queue is the barrier after __set_current_state. */

	__set_current_state(TASK_INTERRUPTIBLE);

	add_wait_queue(&q.waiter, &wait);

	/*

	 * !plist_node_empty() is safe here without any lock.

	 * q.lock_ptr != 0 is not safe, because of ordering against wakeup.

	 */

	if (likely(!plist_node_empty(&q.list))) {

		if (!abs_time)

			schedule();

		else {

			hrtimer_init_on_stack(&t.timer,

					      clockrt ? CLOCK_REALTIME :

					      CLOCK_MONOTONIC,

					      HRTIMER_MODE_ABS);

			hrtimer_init_sleeper(&t, current);

			hrtimer_set_expires_range_ns(&t.timer, *abs_time,

						     current->timer_slack_ns);



			hrtimer_start_expires(&t.timer, HRTIMER_MODE_ABS);

			if (!hrtimer_active(&t.timer))

				t.task = NULL;



			/*

			 * the timer could have already expired, in which

			 * case current would be flagged for rescheduling.

			 * Don't bother calling schedule.

			 */

			if (likely(t.task))

				schedule();



			hrtimer_cancel(&t.timer);



			/* Flag if a timeout occured */

			rem = (t.task == NULL);



			destroy_hrtimer_on_stack(&t.timer);

		}

	}

	__set_current_state(TASK_RUNNING);



	/*

	 * NOTE: we don't remove ourselves from the waitqueue because

	 * we are the only user of it.

	 */

	/* queue_me and wait for wakeup, timeout, or a signal. */

	futex_wait_queue_me(hb, &q, to, &wait);



	/* If we were woken (and unqueued), we succeeded, whatever. */

	ret = 0;

	if (!unqueue_me(&q))

		goto out_put_key;

	ret = -ETIMEDOUT;

	if (rem)

	if (to && !to->task)

		goto out_put_key;



	/*
@@ -1275,6 +1285,10 @@ static int futex_wait(u32 __user *uaddr, int fshared, 
out_put_key:

	put_futex_key(fshared, &q.key);

out:

	if (to) {

		hrtimer_cancel(&to->timer);

		destroy_hrtimer_on_stack(&to->timer);

	}

	return ret;

}