Merge branch 'for-next/futex' of...

the new syscalls yet.

Architectures need to implement the new futex_atomic_cmpxchg_inatomic()

inline function before writing up the syscalls (that function returns

-ENOSYS right now).

inline function before writing up the syscalls.

#include <asm/errno.h>

#define FUTEX_MAX_LOOPS	128 /* What's the largest number you can think of? */

#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg)		\

do {									\

	unsigned int loops = FUTEX_MAX_LOOPS;				\

									\

	uaccess_enable();						\

	asm volatile(							\

"	prfm	pstl1strm, %2\n"					\

"1:	ldxr	%w1, %2\n"						\

	insn "\n"							\

"2:	stlxr	%w3, %w0, %2\n"						\

"	cbnz	%w3, 1b\n"						\

"	dmb	ish\n"							\

"2:	stlxr	%w0, %w3, %2\n"						\

"	cbz	%w0, 3f\n"						\

"	sub	%w4, %w4, %w0\n"					\

"	cbnz	%w4, 1b\n"						\

"	mov	%w0, %w7\n"						\

"3:\n"									\

"	dmb	ish\n"							\

"	.pushsection .fixup,\"ax\"\n"					\

"	.align	2\n"							\

"4:	mov	%w0, %w5\n"						\

"4:	mov	%w0, %w6\n"						\

"	b	3b\n"							\

"	.popsection\n"							\

	_ASM_EXTABLE(1b, 4b)						\

	_ASM_EXTABLE(2b, 4b)						\

	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp)	\

	: "r" (oparg), "Ir" (-EFAULT)					\

	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp),	\

	  "+r" (loops)							\

	: "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN)			\

	: "memory");							\

	uaccess_disable();						\

} while (0)

static inline int

arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)

{

	int oldval = 0, ret, tmp;

	int oldval, ret, tmp;

	u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);

	pagefault_disable();

	switch (op) {

	case FUTEX_OP_SET:

		__futex_atomic_op("mov	%w0, %w4",

		__futex_atomic_op("mov	%w3, %w5",

				  ret, oldval, uaddr, tmp, oparg);

		break;

	case FUTEX_OP_ADD:

		__futex_atomic_op("add	%w0, %w1, %w4",

		__futex_atomic_op("add	%w3, %w1, %w5",

				  ret, oldval, uaddr, tmp, oparg);

		break;

	case FUTEX_OP_OR:

		__futex_atomic_op("orr	%w0, %w1, %w4",

		__futex_atomic_op("orr	%w3, %w1, %w5",

				  ret, oldval, uaddr, tmp, oparg);

		break;

	case FUTEX_OP_ANDN:

		__futex_atomic_op("and	%w0, %w1, %w4",

		__futex_atomic_op("and	%w3, %w1, %w5",

				  ret, oldval, uaddr, tmp, ~oparg);

		break;

	case FUTEX_OP_XOR:

		__futex_atomic_op("eor	%w0, %w1, %w4",

		__futex_atomic_op("eor	%w3, %w1, %w5",

				  ret, oldval, uaddr, tmp, oparg);

		break;

	default:

			      u32 oldval, u32 newval)

{

	int ret = 0;

	unsigned int loops = FUTEX_MAX_LOOPS;

	u32 val, tmp;

	u32 __user *uaddr;

	asm volatile("// futex_atomic_cmpxchg_inatomic\n"

"	prfm	pstl1strm, %2\n"

"1:	ldxr	%w1, %2\n"

"	sub	%w3, %w1, %w4\n"

"	cbnz	%w3, 3f\n"

"2:	stlxr	%w3, %w5, %2\n"

"	cbnz	%w3, 1b\n"

"	dmb	ish\n"

"	sub	%w3, %w1, %w5\n"

"	cbnz	%w3, 4f\n"

"2:	stlxr	%w3, %w6, %2\n"

"	cbz	%w3, 3f\n"

"	sub	%w4, %w4, %w3\n"

"	cbnz	%w4, 1b\n"

"	mov	%w0, %w8\n"

"3:\n"

"	dmb	ish\n"

"4:\n"

"	.pushsection .fixup,\"ax\"\n"

"4:	mov	%w0, %w6\n"

"	b	3b\n"

"5:	mov	%w0, %w7\n"

"	b	4b\n"

"	.popsection\n"

	_ASM_EXTABLE(1b, 4b)

	_ASM_EXTABLE(2b, 4b)

	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)

	: "r" (oldval), "r" (newval), "Ir" (-EFAULT)

	_ASM_EXTABLE(1b, 5b)

	_ASM_EXTABLE(2b, 5b)

	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)

	: "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)

	: "memory");

	uaccess_disable();

	if (!ret)

		*uval = val;

	return ret;

}

 *

 * Return:

 * 0 - On success

 * <0 - On error

 * -EFAULT - User access resulted in a page fault

 * -EAGAIN - Atomic operation was unable to complete due to contention

 * -ENOSYS - Operation not supported

 */

static inline int

arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr)

 *

 * Return:

 * 0 - On success

 * <0 - On error

 * -EFAULT - User access resulted in a page fault

 * -EAGAIN - Atomic operation was unable to complete due to contention

 * -ENOSYS - Function not implemented (only if !HAVE_FUTEX_CMPXCHG)

 */

static inline int

futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,

static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)

{

	int err;

	u32 uninitialized_var(curval);

	if (unlikely(should_fail_futex(true)))

		return -EFAULT;

	if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))

		return -EFAULT;

	err = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);

	if (unlikely(err))

		return err;

	/* If user space value changed, let the caller retry */

	return curval != uval ? -EAGAIN : 0;

	if (unlikely(should_fail_futex(true)))

		ret = -EFAULT;

	if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)) {

		ret = -EFAULT;

	} else if (curval != uval) {

	ret = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);

	if (!ret && (curval != uval)) {

		/*

		 * If a unconditional UNLOCK_PI operation (user space did not

		 * try the TID->0 transition) raced with a waiter setting the

	double_lock_hb(hb1, hb2);

	op_ret = futex_atomic_op_inuser(op, uaddr2);

	if (unlikely(op_ret < 0)) {

		double_unlock_hb(hb1, hb2);

#ifndef CONFIG_MMU

		if (!IS_ENABLED(CONFIG_MMU) ||

		    unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {

			/*

		 * we don't get EFAULT from MMU faults if we don't have an MMU,

		 * but we might get them from range checking

			 * we don't get EFAULT from MMU faults if we don't have

			 * an MMU, but we might get them from range checking

			 */

			ret = op_ret;

			goto out_put_keys;

#endif

		if (unlikely(op_ret != -EFAULT)) {

			ret = op_ret;

			goto out_put_keys;

		}

		if (op_ret == -EFAULT) {

			ret = fault_in_user_writeable(uaddr2);

			if (ret)

				goto out_put_keys;

		}

		if (!(flags & FLAGS_SHARED))

		if (!(flags & FLAGS_SHARED)) {

			cond_resched();

			goto retry_private;

		}

		put_futex_key(&key2);

		put_futex_key(&key1);

		cond_resched();

		goto retry;

	}

	u32 uval, uninitialized_var(curval), newval;

	struct task_struct *oldowner, *newowner;

	u32 newtid;

	int ret;

	int ret, err = 0;

	lockdep_assert_held(q->lock_ptr);

	if (!pi_state->owner)

		newtid |= FUTEX_OWNER_DIED;

	if (get_futex_value_locked(&uval, uaddr))

		goto handle_fault;

	err = get_futex_value_locked(&uval, uaddr);

	if (err)

		goto handle_err;

	for (;;) {

		newval = (uval & FUTEX_OWNER_DIED) | newtid;

		if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))

			goto handle_fault;

		err = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);

		if (err)

			goto handle_err;

		if (curval == uval)

			break;

		uval = curval;

	return 0;

	/*

	 * To handle the page fault we need to drop the locks here. That gives

	 * the other task (either the highest priority waiter itself or the

	 * task which stole the rtmutex) the chance to try the fixup of the

	 * pi_state. So once we are back from handling the fault we need to

	 * check the pi_state after reacquiring the locks and before trying to

	 * do another fixup. When the fixup has been done already we simply

	 * return.

	 * In order to reschedule or handle a page fault, we need to drop the

	 * locks here. In the case of a fault, this gives the other task

	 * (either the highest priority waiter itself or the task which stole

	 * the rtmutex) the chance to try the fixup of the pi_state. So once we

	 * are back from handling the fault we need to check the pi_state after

	 * reacquiring the locks and before trying to do another fixup. When

	 * the fixup has been done already we simply return.

	 *

	 * Note: we hold both hb->lock and pi_mutex->wait_lock. We can safely

	 * drop hb->lock since the caller owns the hb -> futex_q relation.

	 * Dropping the pi_mutex->wait_lock requires the state revalidate.

	 */

handle_fault:

handle_err:

	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);

	spin_unlock(q->lock_ptr);

	switch (err) {

	case -EFAULT:

		ret = fault_in_user_writeable(uaddr);

		break;

	case -EAGAIN:

		cond_resched();

		ret = 0;

		break;

	default:

		WARN_ON_ONCE(1);

		ret = err;

		break;

	}

	spin_lock(q->lock_ptr);

	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);

		 * A unconditional UNLOCK_PI op raced against a waiter

		 * setting the FUTEX_WAITERS bit. Try again.

		 */

		if (ret == -EAGAIN) {

			put_futex_key(&key);

			goto retry;

		}

		if (ret == -EAGAIN)

			goto pi_retry;

		/*

		 * wake_futex_pi has detected invalid state. Tell user

		 * space.

	 * preserve the WAITERS bit not the OWNER_DIED one. We are the

	 * owner.

	 */

	if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0)) {

	if ((ret = cmpxchg_futex_value_locked(&curval, uaddr, uval, 0))) {

		spin_unlock(&hb->lock);

		switch (ret) {

		case -EFAULT:

			goto pi_faulted;

		case -EAGAIN:

			goto pi_retry;

		default:

			WARN_ON_ONCE(1);

			goto out_putkey;

		}

	}

	/*

	put_futex_key(&key);

	return ret;

pi_retry:

	put_futex_key(&key);

	cond_resched();

	goto retry;

pi_faulted:

	put_futex_key(&key);

static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, int pi)

{

	u32 uval, uninitialized_var(nval), mval;

	int err;

	/* Futex address must be 32bit aligned */

	if ((((unsigned long)uaddr) % sizeof(*uaddr)) != 0)

	if (get_user(uval, uaddr))

		return -1;

	if ((uval & FUTEX_TID_MASK) == task_pid_vnr(curr)) {

	if ((uval & FUTEX_TID_MASK) != task_pid_vnr(curr))

		return 0;

	/*

	 * Ok, this dying thread is truly holding a futex

	 * of interest. Set the OWNER_DIED bit atomically

	 * userspace.

	 */

	mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;

	/*

	 * We are not holding a lock here, but we want to have

	 * the pagefault_disable/enable() protection because

	 * does not guarantee R/W access. If that fails we

	 * give up and leave the futex locked.

	 */

		if (cmpxchg_futex_value_locked(&nval, uaddr, uval, mval)) {

	if ((err = cmpxchg_futex_value_locked(&nval, uaddr, uval, mval))) {

		switch (err) {

		case -EFAULT:

			if (fault_in_user_writeable(uaddr))

				return -1;

			goto retry;

		case -EAGAIN:

			cond_resched();

			goto retry;

		default:

			WARN_ON_ONCE(1);

			return err;

		}

	}

	if (nval != uval)

		goto retry;

	 */

	if (!pi && (uval & FUTEX_WAITERS))

		futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);

	}

	return 0;

}