Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 36e91aa2 authored by Ingo Molnar's avatar Ingo Molnar
Browse files

Merge branch 'locking/arch-atomic' into locking/core, because the topic is ready 



Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parents 03e3c2b7 b7271b9f

arch/alpha/include/asm/atomic.h

+72 −15
Original line number Diff line number Diff line
@@ -46,10 +46,9 @@ static __inline__ void atomic_##op(int i, atomic_t * v) \ 
}									\



#define ATOMIC_OP_RETURN(op, asm_op)					\

static inline int atomic_##op##_return(int i, atomic_t *v)		\

static inline int atomic_##op##_return_relaxed(int i, atomic_t *v)	\

{									\

	long temp, result;						\

	smp_mb();							\

	__asm__ __volatile__(						\

	"1:	ldl_l %0,%1\n"						\

	"	" #asm_op " %0,%3,%2\n"					\
@@ -61,7 +60,23 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ 
	".previous"							\

	:"=&r" (temp), "=m" (v->counter), "=&r" (result)		\

	:"Ir" (i), "m" (v->counter) : "memory");			\

	smp_mb();							\

	return result;							\

}



#define ATOMIC_FETCH_OP(op, asm_op)					\

static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\

{									\

	long temp, result;						\

	__asm__ __volatile__(						\

	"1:	ldl_l %2,%1\n"						\

	"	" #asm_op " %2,%3,%0\n"					\

	"	stl_c %0,%1\n"						\

	"	beq %0,2f\n"						\

	".subsection 2\n"						\

	"2:	br 1b\n"						\

	".previous"							\

	:"=&r" (temp), "=m" (v->counter), "=&r" (result)		\

	:"Ir" (i), "m" (v->counter) : "memory");			\

	return result;							\

}
@@ -82,10 +97,9 @@ static __inline__ void atomic64_##op(long i, atomic64_t * v) \ 
}									\



#define ATOMIC64_OP_RETURN(op, asm_op)					\

static __inline__ long atomic64_##op##_return(long i, atomic64_t * v)	\

static __inline__ long atomic64_##op##_return_relaxed(long i, atomic64_t * v)	\

{									\

	long temp, result;						\

	smp_mb();							\

	__asm__ __volatile__(						\

	"1:	ldq_l %0,%1\n"						\

	"	" #asm_op " %0,%3,%2\n"					\
@@ -97,34 +111,77 @@ static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ 
	".previous"							\

	:"=&r" (temp), "=m" (v->counter), "=&r" (result)		\

	:"Ir" (i), "m" (v->counter) : "memory");			\

	smp_mb();							\

	return result;							\

}



#define ATOMIC64_FETCH_OP(op, asm_op)					\

static __inline__ long atomic64_fetch_##op##_relaxed(long i, atomic64_t * v)	\

{									\

	long temp, result;						\

	__asm__ __volatile__(						\

	"1:	ldq_l %2,%1\n"						\

	"	" #asm_op " %2,%3,%0\n"					\

	"	stq_c %0,%1\n"						\

	"	beq %0,2f\n"						\

	".subsection 2\n"						\

	"2:	br 1b\n"						\

	".previous"							\

	:"=&r" (temp), "=m" (v->counter), "=&r" (result)		\

	:"Ir" (i), "m" (v->counter) : "memory");			\

	return result;							\

}



#define ATOMIC_OPS(op)							\

	ATOMIC_OP(op, op##l)						\

	ATOMIC_OP_RETURN(op, op##l)					\

	ATOMIC_FETCH_OP(op, op##l)					\

	ATOMIC64_OP(op, op##q)						\

	ATOMIC64_OP_RETURN(op, op##q)

	ATOMIC64_OP_RETURN(op, op##q)					\

	ATOMIC64_FETCH_OP(op, op##q)



ATOMIC_OPS(add)

ATOMIC_OPS(sub)



#define atomic_add_return_relaxed	atomic_add_return_relaxed

#define atomic_sub_return_relaxed	atomic_sub_return_relaxed

#define atomic_fetch_add_relaxed	atomic_fetch_add_relaxed

#define atomic_fetch_sub_relaxed	atomic_fetch_sub_relaxed



#define atomic64_add_return_relaxed	atomic64_add_return_relaxed

#define atomic64_sub_return_relaxed	atomic64_sub_return_relaxed

#define atomic64_fetch_add_relaxed	atomic64_fetch_add_relaxed

#define atomic64_fetch_sub_relaxed	atomic64_fetch_sub_relaxed



#define atomic_andnot atomic_andnot

#define atomic64_andnot atomic64_andnot



ATOMIC_OP(and, and)

ATOMIC_OP(andnot, bic)

ATOMIC_OP(or, bis)

ATOMIC_OP(xor, xor)

ATOMIC64_OP(and, and)

ATOMIC64_OP(andnot, bic)

ATOMIC64_OP(or, bis)

ATOMIC64_OP(xor, xor)

#undef ATOMIC_OPS

#define ATOMIC_OPS(op, asm)						\

	ATOMIC_OP(op, asm)						\

	ATOMIC_FETCH_OP(op, asm)					\

	ATOMIC64_OP(op, asm)						\

	ATOMIC64_FETCH_OP(op, asm)



ATOMIC_OPS(and, and)

ATOMIC_OPS(andnot, bic)

ATOMIC_OPS(or, bis)

ATOMIC_OPS(xor, xor)



#define atomic_fetch_and_relaxed	atomic_fetch_and_relaxed

#define atomic_fetch_andnot_relaxed	atomic_fetch_andnot_relaxed

#define atomic_fetch_or_relaxed		atomic_fetch_or_relaxed

#define atomic_fetch_xor_relaxed	atomic_fetch_xor_relaxed



#define atomic64_fetch_and_relaxed	atomic64_fetch_and_relaxed

#define atomic64_fetch_andnot_relaxed	atomic64_fetch_andnot_relaxed

#define atomic64_fetch_or_relaxed	atomic64_fetch_or_relaxed

#define atomic64_fetch_xor_relaxed	atomic64_fetch_xor_relaxed



#undef ATOMIC_OPS

#undef ATOMIC64_FETCH_OP

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP

arch/arc/include/asm/atomic.h

+87 −12
Original line number Diff line number Diff line
@@ -67,6 +67,33 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ 
	return val;							\

}



#define ATOMIC_FETCH_OP(op, c_op, asm_op)				\

static inline int atomic_fetch_##op(int i, atomic_t *v)			\

{									\

	unsigned int val, orig;						\

									\

	/*								\

	 * Explicit full memory barrier needed before/after as		\

	 * LLOCK/SCOND thmeselves don't provide any such semantics	\

	 */								\

	smp_mb();							\

									\

	__asm__ __volatile__(						\

	"1:	llock   %[orig], [%[ctr]]		\n"		\

	"	" #asm_op " %[val], %[orig], %[i]	\n"		\

	"	scond   %[val], [%[ctr]]		\n"		\

	"						\n"		\

	: [val]	"=&r"	(val),						\

	  [orig] "=&r" (orig)						\

	: [ctr]	"r"	(&v->counter),					\

	  [i]	"ir"	(i)						\

	: "cc");							\

									\

	smp_mb();							\

									\

	return orig;							\

}



#else	/* !CONFIG_ARC_HAS_LLSC */



#ifndef CONFIG_SMP
@@ -129,25 +156,44 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ 
	return temp;							\

}



#define ATOMIC_FETCH_OP(op, c_op, asm_op)				\

static inline int atomic_fetch_##op(int i, atomic_t *v)			\

{									\

	unsigned long flags;						\

	unsigned long orig;						\

									\

	/*								\

	 * spin lock/unlock provides the needed smp_mb() before/after	\

	 */								\

	atomic_ops_lock(flags);						\

	orig = v->counter;						\

	v->counter c_op i;						\

	atomic_ops_unlock(flags);					\

									\

	return orig;							\

}



#endif /* !CONFIG_ARC_HAS_LLSC */



#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_OP_RETURN(op, c_op, asm_op)

	ATOMIC_OP_RETURN(op, c_op, asm_op)				\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(add, +=, add)

ATOMIC_OPS(sub, -=, sub)



#define atomic_andnot atomic_andnot



ATOMIC_OP(and, &=, and)

ATOMIC_OP(andnot, &= ~, bic)

ATOMIC_OP(or, |=, or)

ATOMIC_OP(xor, ^=, xor)

#undef ATOMIC_OPS

#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



#undef SCOND_FAIL_RETRY_VAR_DEF

#undef SCOND_FAIL_RETRY_ASM

#undef SCOND_FAIL_RETRY_VARS

ATOMIC_OPS(and, &=, and)

ATOMIC_OPS(andnot, &= ~, bic)

ATOMIC_OPS(or, |=, or)

ATOMIC_OPS(xor, ^=, xor)



#else /* CONFIG_ARC_PLAT_EZNPS */
@@ -208,22 +254,51 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ 
	return temp;							\

}



#define ATOMIC_FETCH_OP(op, c_op, asm_op)				\

static inline int atomic_fetch_##op(int i, atomic_t *v)			\

{									\

	unsigned int temp = i;						\

									\

	/* Explicit full memory barrier needed before/after */		\

	smp_mb();							\

									\

	__asm__ __volatile__(						\

	"	mov r2, %0\n"						\

	"	mov r3, %1\n"						\

	"       .word %2\n"						\

	"	mov %0, r2"						\

	: "+r"(temp)							\

	: "r"(&v->counter), "i"(asm_op)					\

	: "r2", "r3", "memory");					\

									\

	smp_mb();							\

									\

	return temp;							\

}



#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_OP_RETURN(op, c_op, asm_op)

	ATOMIC_OP_RETURN(op, c_op, asm_op)				\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(add, +=, CTOP_INST_AADD_DI_R2_R2_R3)

#define atomic_sub(i, v) atomic_add(-(i), (v))

#define atomic_sub_return(i, v) atomic_add_return(-(i), (v))



ATOMIC_OP(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)

#undef ATOMIC_OPS

#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(and, &=, CTOP_INST_AAND_DI_R2_R2_R3)

#define atomic_andnot(mask, v) atomic_and(~(mask), (v))

ATOMIC_OP(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)

ATOMIC_OP(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)

ATOMIC_OPS(or, |=, CTOP_INST_AOR_DI_R2_R2_R3)

ATOMIC_OPS(xor, ^=, CTOP_INST_AXOR_DI_R2_R2_R3)



#endif /* CONFIG_ARC_PLAT_EZNPS */



#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP

arch/arm/include/asm/atomic.h

+96 −10
Original line number Diff line number Diff line
@@ -77,8 +77,36 @@ static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ 
	return result;							\

}



#define ATOMIC_FETCH_OP(op, c_op, asm_op)				\

static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v)	\

{									\

	unsigned long tmp;						\

	int result, val;						\

									\

	prefetchw(&v->counter);						\

									\

	__asm__ __volatile__("@ atomic_fetch_" #op "\n"			\

"1:	ldrex	%0, [%4]\n"						\

"	" #asm_op "	%1, %0, %5\n"					\

"	strex	%2, %1, [%4]\n"						\

"	teq	%2, #0\n"						\

"	bne	1b"							\

	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Qo" (v->counter)	\

	: "r" (&v->counter), "Ir" (i)					\

	: "cc");							\

									\

	return result;							\

}



#define atomic_add_return_relaxed	atomic_add_return_relaxed

#define atomic_sub_return_relaxed	atomic_sub_return_relaxed

#define atomic_fetch_add_relaxed	atomic_fetch_add_relaxed

#define atomic_fetch_sub_relaxed	atomic_fetch_sub_relaxed



#define atomic_fetch_and_relaxed	atomic_fetch_and_relaxed

#define atomic_fetch_andnot_relaxed	atomic_fetch_andnot_relaxed

#define atomic_fetch_or_relaxed		atomic_fetch_or_relaxed

#define atomic_fetch_xor_relaxed	atomic_fetch_xor_relaxed



static inline int atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new)

{
@@ -159,6 +187,20 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ 
	return val;							\

}



#define ATOMIC_FETCH_OP(op, c_op, asm_op)				\

static inline int atomic_fetch_##op(int i, atomic_t *v)			\

{									\

	unsigned long flags;						\

	int val;							\

									\

	raw_local_irq_save(flags);					\

	val = v->counter;						\

	v->counter c_op i;						\

	raw_local_irq_restore(flags);					\

									\

	return val;							\

}



static inline int atomic_cmpxchg(atomic_t *v, int old, int new)

{

	int ret;
@@ -187,19 +229,26 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) 


#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_OP_RETURN(op, c_op, asm_op)

	ATOMIC_OP_RETURN(op, c_op, asm_op)				\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(add, +=, add)

ATOMIC_OPS(sub, -=, sub)



#define atomic_andnot atomic_andnot



ATOMIC_OP(and, &=, and)

ATOMIC_OP(andnot, &= ~, bic)

ATOMIC_OP(or,  |=, orr)

ATOMIC_OP(xor, ^=, eor)

#undef ATOMIC_OPS

#define ATOMIC_OPS(op, c_op, asm_op)					\

	ATOMIC_OP(op, c_op, asm_op)					\

	ATOMIC_FETCH_OP(op, c_op, asm_op)



ATOMIC_OPS(and, &=, and)

ATOMIC_OPS(andnot, &= ~, bic)

ATOMIC_OPS(or,  |=, orr)

ATOMIC_OPS(xor, ^=, eor)



#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP
@@ -317,24 +366,61 @@ atomic64_##op##_return_relaxed(long long i, atomic64_t *v) \ 
	return result;							\

}



#define ATOMIC64_FETCH_OP(op, op1, op2)					\

static inline long long							\

atomic64_fetch_##op##_relaxed(long long i, atomic64_t *v)		\

{									\

	long long result, val;						\

	unsigned long tmp;						\

									\

	prefetchw(&v->counter);						\

									\

	__asm__ __volatile__("@ atomic64_fetch_" #op "\n"		\

"1:	ldrexd	%0, %H0, [%4]\n"					\

"	" #op1 " %Q1, %Q0, %Q5\n"					\

"	" #op2 " %R1, %R0, %R5\n"					\

"	strexd	%2, %1, %H1, [%4]\n"					\

"	teq	%2, #0\n"						\

"	bne	1b"							\

	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Qo" (v->counter)	\

	: "r" (&v->counter), "r" (i)					\

	: "cc");							\

									\

	return result;							\

}



#define ATOMIC64_OPS(op, op1, op2)					\

	ATOMIC64_OP(op, op1, op2)					\

	ATOMIC64_OP_RETURN(op, op1, op2)

	ATOMIC64_OP_RETURN(op, op1, op2)				\

	ATOMIC64_FETCH_OP(op, op1, op2)



ATOMIC64_OPS(add, adds, adc)

ATOMIC64_OPS(sub, subs, sbc)



#define atomic64_add_return_relaxed	atomic64_add_return_relaxed

#define atomic64_sub_return_relaxed	atomic64_sub_return_relaxed

#define atomic64_fetch_add_relaxed	atomic64_fetch_add_relaxed

#define atomic64_fetch_sub_relaxed	atomic64_fetch_sub_relaxed



#undef ATOMIC64_OPS

#define ATOMIC64_OPS(op, op1, op2)					\

	ATOMIC64_OP(op, op1, op2)					\

	ATOMIC64_FETCH_OP(op, op1, op2)



#define atomic64_andnot atomic64_andnot



ATOMIC64_OP(and, and, and)

ATOMIC64_OP(andnot, bic, bic)

ATOMIC64_OP(or,  orr, orr)

ATOMIC64_OP(xor, eor, eor)

ATOMIC64_OPS(and, and, and)

ATOMIC64_OPS(andnot, bic, bic)

ATOMIC64_OPS(or,  orr, orr)

ATOMIC64_OPS(xor, eor, eor)



#define atomic64_fetch_and_relaxed	atomic64_fetch_and_relaxed

#define atomic64_fetch_andnot_relaxed	atomic64_fetch_andnot_relaxed

#define atomic64_fetch_or_relaxed	atomic64_fetch_or_relaxed

#define atomic64_fetch_xor_relaxed	atomic64_fetch_xor_relaxed



#undef ATOMIC64_OPS

#undef ATOMIC64_FETCH_OP

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP

arch/arm64/include/asm/atomic.h

+60 −0
Original line number Diff line number Diff line
@@ -76,6 +76,36 @@ 
#define atomic_dec_return_release(v)	atomic_sub_return_release(1, (v))

#define atomic_dec_return(v)		atomic_sub_return(1, (v))



#define atomic_fetch_add_relaxed	atomic_fetch_add_relaxed

#define atomic_fetch_add_acquire	atomic_fetch_add_acquire

#define atomic_fetch_add_release	atomic_fetch_add_release

#define atomic_fetch_add		atomic_fetch_add



#define atomic_fetch_sub_relaxed	atomic_fetch_sub_relaxed

#define atomic_fetch_sub_acquire	atomic_fetch_sub_acquire

#define atomic_fetch_sub_release	atomic_fetch_sub_release

#define atomic_fetch_sub		atomic_fetch_sub



#define atomic_fetch_and_relaxed	atomic_fetch_and_relaxed

#define atomic_fetch_and_acquire	atomic_fetch_and_acquire

#define atomic_fetch_and_release	atomic_fetch_and_release

#define atomic_fetch_and		atomic_fetch_and



#define atomic_fetch_andnot_relaxed	atomic_fetch_andnot_relaxed

#define atomic_fetch_andnot_acquire	atomic_fetch_andnot_acquire

#define atomic_fetch_andnot_release	atomic_fetch_andnot_release

#define atomic_fetch_andnot		atomic_fetch_andnot



#define atomic_fetch_or_relaxed		atomic_fetch_or_relaxed

#define atomic_fetch_or_acquire		atomic_fetch_or_acquire

#define atomic_fetch_or_release		atomic_fetch_or_release

#define atomic_fetch_or			atomic_fetch_or



#define atomic_fetch_xor_relaxed	atomic_fetch_xor_relaxed

#define atomic_fetch_xor_acquire	atomic_fetch_xor_acquire

#define atomic_fetch_xor_release	atomic_fetch_xor_release

#define atomic_fetch_xor		atomic_fetch_xor



#define atomic_xchg_relaxed(v, new)	xchg_relaxed(&((v)->counter), (new))

#define atomic_xchg_acquire(v, new)	xchg_acquire(&((v)->counter), (new))

#define atomic_xchg_release(v, new)	xchg_release(&((v)->counter), (new))
@@ -125,6 +155,36 @@ 
#define atomic64_dec_return_release(v)	atomic64_sub_return_release(1, (v))

#define atomic64_dec_return(v)		atomic64_sub_return(1, (v))



#define atomic64_fetch_add_relaxed	atomic64_fetch_add_relaxed

#define atomic64_fetch_add_acquire	atomic64_fetch_add_acquire

#define atomic64_fetch_add_release	atomic64_fetch_add_release

#define atomic64_fetch_add		atomic64_fetch_add



#define atomic64_fetch_sub_relaxed	atomic64_fetch_sub_relaxed

#define atomic64_fetch_sub_acquire	atomic64_fetch_sub_acquire

#define atomic64_fetch_sub_release	atomic64_fetch_sub_release

#define atomic64_fetch_sub		atomic64_fetch_sub



#define atomic64_fetch_and_relaxed	atomic64_fetch_and_relaxed

#define atomic64_fetch_and_acquire	atomic64_fetch_and_acquire

#define atomic64_fetch_and_release	atomic64_fetch_and_release

#define atomic64_fetch_and		atomic64_fetch_and



#define atomic64_fetch_andnot_relaxed	atomic64_fetch_andnot_relaxed

#define atomic64_fetch_andnot_acquire	atomic64_fetch_andnot_acquire

#define atomic64_fetch_andnot_release	atomic64_fetch_andnot_release

#define atomic64_fetch_andnot		atomic64_fetch_andnot



#define atomic64_fetch_or_relaxed	atomic64_fetch_or_relaxed

#define atomic64_fetch_or_acquire	atomic64_fetch_or_acquire

#define atomic64_fetch_or_release	atomic64_fetch_or_release

#define atomic64_fetch_or		atomic64_fetch_or



#define atomic64_fetch_xor_relaxed	atomic64_fetch_xor_relaxed

#define atomic64_fetch_xor_acquire	atomic64_fetch_xor_acquire

#define atomic64_fetch_xor_release	atomic64_fetch_xor_release

#define atomic64_fetch_xor		atomic64_fetch_xor



#define atomic64_xchg_relaxed		atomic_xchg_relaxed

#define atomic64_xchg_acquire		atomic_xchg_acquire

#define atomic64_xchg_release		atomic_xchg_release

arch/arm64/include/asm/atomic_ll_sc.h

+86 −24
Original line number Diff line number Diff line
@@ -77,26 +77,57 @@ __LL_SC_PREFIX(atomic_##op##_return##name(int i, atomic_t *v)) \ 
}									\

__LL_SC_EXPORT(atomic_##op##_return##name);



#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op)		\

__LL_SC_INLINE int							\

__LL_SC_PREFIX(atomic_fetch_##op##name(int i, atomic_t *v))		\

{									\

	unsigned long tmp;						\

	int val, result;						\

									\

	asm volatile("// atomic_fetch_" #op #name "\n"			\

"	prfm	pstl1strm, %3\n"					\

"1:	ld" #acq "xr	%w0, %3\n"					\

"	" #asm_op "	%w1, %w0, %w4\n"				\

"	st" #rel "xr	%w2, %w1, %3\n"					\

"	cbnz	%w2, 1b\n"						\

"	" #mb								\

	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\

	: "Ir" (i)							\

	: cl);								\

									\

	return result;							\

}									\

__LL_SC_EXPORT(atomic_fetch_##op##name);



#define ATOMIC_OPS(...)							\

	ATOMIC_OP(__VA_ARGS__)						\

	ATOMIC_OP_RETURN(        , dmb ish,  , l, "memory", __VA_ARGS__)



#define ATOMIC_OPS_RLX(...)						\

	ATOMIC_OPS(__VA_ARGS__)						\

	ATOMIC_OP_RETURN(        , dmb ish,  , l, "memory", __VA_ARGS__)\

	ATOMIC_OP_RETURN(_relaxed,        ,  ,  ,         , __VA_ARGS__)\

	ATOMIC_OP_RETURN(_acquire,        , a,  , "memory", __VA_ARGS__)\

	ATOMIC_OP_RETURN(_release,        ,  , l, "memory", __VA_ARGS__)

	ATOMIC_OP_RETURN(_release,        ,  , l, "memory", __VA_ARGS__)\

	ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\

	ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\

	ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\

	ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)



ATOMIC_OPS_RLX(add, add)

ATOMIC_OPS_RLX(sub, sub)

ATOMIC_OPS(add, add)

ATOMIC_OPS(sub, sub)



#undef ATOMIC_OPS

#define ATOMIC_OPS(...)							\

	ATOMIC_OP(__VA_ARGS__)						\

	ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\

	ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\

	ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\

	ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)



ATOMIC_OP(and, and)

ATOMIC_OP(andnot, bic)

ATOMIC_OP(or, orr)

ATOMIC_OP(xor, eor)

ATOMIC_OPS(and, and)

ATOMIC_OPS(andnot, bic)

ATOMIC_OPS(or, orr)

ATOMIC_OPS(xor, eor)



#undef ATOMIC_OPS_RLX

#undef ATOMIC_OPS

#undef ATOMIC_FETCH_OP

#undef ATOMIC_OP_RETURN

#undef ATOMIC_OP
@@ -140,26 +171,57 @@ __LL_SC_PREFIX(atomic64_##op##_return##name(long i, atomic64_t *v)) \ 
}									\

__LL_SC_EXPORT(atomic64_##op##_return##name);



#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op)		\

__LL_SC_INLINE long							\

__LL_SC_PREFIX(atomic64_fetch_##op##name(long i, atomic64_t *v))	\

{									\

	long result, val;						\

	unsigned long tmp;						\

									\

	asm volatile("// atomic64_fetch_" #op #name "\n"		\

"	prfm	pstl1strm, %3\n"					\

"1:	ld" #acq "xr	%0, %3\n"					\

"	" #asm_op "	%1, %0, %4\n"					\

"	st" #rel "xr	%w2, %1, %3\n"					\

"	cbnz	%w2, 1b\n"						\

"	" #mb								\

	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\

	: "Ir" (i)							\

	: cl);								\

									\

	return result;							\

}									\

__LL_SC_EXPORT(atomic64_fetch_##op##name);



#define ATOMIC64_OPS(...)						\

	ATOMIC64_OP(__VA_ARGS__)					\

	ATOMIC64_OP_RETURN(, dmb ish,  , l, "memory", __VA_ARGS__)



#define ATOMIC64_OPS_RLX(...)						\

	ATOMIC64_OPS(__VA_ARGS__)					\

	ATOMIC64_OP_RETURN(, dmb ish,  , l, "memory", __VA_ARGS__)	\

	ATOMIC64_OP_RETURN(_relaxed,,  ,  ,         , __VA_ARGS__)	\

	ATOMIC64_OP_RETURN(_acquire,, a,  , "memory", __VA_ARGS__)	\

	ATOMIC64_OP_RETURN(_release,,  , l, "memory", __VA_ARGS__)

	ATOMIC64_OP_RETURN(_release,,  , l, "memory", __VA_ARGS__)	\

	ATOMIC64_FETCH_OP (, dmb ish,  , l, "memory", __VA_ARGS__)	\

	ATOMIC64_FETCH_OP (_relaxed,,  ,  ,         , __VA_ARGS__)	\

	ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)	\

	ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)



ATOMIC64_OPS_RLX(add, add)

ATOMIC64_OPS_RLX(sub, sub)

ATOMIC64_OPS(add, add)

ATOMIC64_OPS(sub, sub)



#undef ATOMIC64_OPS

#define ATOMIC64_OPS(...)						\

	ATOMIC64_OP(__VA_ARGS__)					\

	ATOMIC64_FETCH_OP (, dmb ish,  , l, "memory", __VA_ARGS__)	\

	ATOMIC64_FETCH_OP (_relaxed,,  ,  ,         , __VA_ARGS__)	\

	ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)	\

	ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)



ATOMIC64_OP(and, and)

ATOMIC64_OP(andnot, bic)

ATOMIC64_OP(or, orr)

ATOMIC64_OP(xor, eor)

ATOMIC64_OPS(and, and)

ATOMIC64_OPS(andnot, bic)

ATOMIC64_OPS(or, orr)

ATOMIC64_OPS(xor, eor)



#undef ATOMIC64_OPS_RLX

#undef ATOMIC64_OPS

#undef ATOMIC64_FETCH_OP

#undef ATOMIC64_OP_RETURN

#undef ATOMIC64_OP