Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 03e3c2b7 authored by Will Deacon's avatar Will Deacon Committed by Ingo Molnar
Browse files

locking/barriers, arch/arm64: Implement LDXR+WFE based smp_cond_load_acquire() 



smp_cond_load_acquire() is used to spin on a variable until some
expression involving that variable becomes true.

On arm64, we can build this using the LDXR and WFE instructions, since
clearing of the exclusive monitor as a result of the variable being
changed by another CPU generates an event, which will wake us up out of WFE.

This patch implements smp_cond_load_acquire() using LDXR and WFE, which
themselves are contained in an internal __cmpwait() function.

Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: catalin.marinas@arm.com
Cc: linux-arm-kernel@lists.infradead.org
Link: http://lkml.kernel.org/r/1467049434-30451-1-git-send-email-will.deacon@arm.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 885885f6

arch/arm64/include/asm/barrier.h

+13 −0
Original line number Diff line number Diff line
@@ -91,6 +91,19 @@ do { \ 
	__u.__val;							\

})



#define smp_cond_load_acquire(ptr, cond_expr)				\

({									\

	typeof(ptr) __PTR = (ptr);					\

	typeof(*ptr) VAL;						\

	for (;;) {							\

		VAL = smp_load_acquire(__PTR);				\

		if (cond_expr)						\

			break;						\

		__cmpwait_relaxed(__PTR, VAL);				\

	}								\

	VAL;								\

})



#include <asm-generic/barrier.h>



#endif	/* __ASSEMBLY__ */

arch/arm64/include/asm/cmpxchg.h

+51 −0
Original line number Diff line number Diff line
@@ -224,4 +224,55 @@ __CMPXCHG_GEN(_mb) 
	__ret;								\

})



#define __CMPWAIT_CASE(w, sz, name)					\

static inline void __cmpwait_case_##name(volatile void *ptr,		\

					 unsigned long val)		\

{									\

	unsigned long tmp;						\

									\

	asm volatile(							\

	"	ldxr" #sz "\t%" #w "[tmp], %[v]\n"		\

	"	eor	%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n"	\

	"	cbnz	%" #w "[tmp], 1f\n"				\

	"	wfe\n"							\

	"1:"								\

	: [tmp] "=&r" (tmp), [v] "+Q" (*(unsigned long *)ptr)		\

	: [val] "r" (val));						\

}



__CMPWAIT_CASE(w, b, 1);

__CMPWAIT_CASE(w, h, 2);

__CMPWAIT_CASE(w,  , 4);

__CMPWAIT_CASE( ,  , 8);



#undef __CMPWAIT_CASE



#define __CMPWAIT_GEN(sfx)						\

static inline void __cmpwait##sfx(volatile void *ptr,			\

				  unsigned long val,			\

				  int size)				\

{									\

	switch (size) {							\

	case 1:								\

		return __cmpwait_case##sfx##_1(ptr, (u8)val);		\

	case 2:								\

		return __cmpwait_case##sfx##_2(ptr, (u16)val);		\

	case 4:								\

		return __cmpwait_case##sfx##_4(ptr, val);		\

	case 8:								\

		return __cmpwait_case##sfx##_8(ptr, val);		\

	default:							\

		BUILD_BUG();						\

	}								\

									\

	unreachable();							\

}



__CMPWAIT_GEN()



#undef __CMPWAIT_GEN



#define __cmpwait_relaxed(ptr, val) \

	__cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr)))



#endif	/* __ASM_CMPXCHG_H */