Merge branch 'for-next/atomics' into for-next/core (61b7cddf) · Commits · e / devices / android_kernel_fairphone_FP5

arch/arm64/Kconfig

+1 −0

Original line number	Diff line number	Diff line
		@@ -1290,6 +1290,7 @@ config ARM64_PAN

		config ARM64_LSE_ATOMICS
		bool "Atomic instructions"
		depends on JUMP_LABEL
		default y
		help
		As part of the Large System Extensions, ARMv8.1 introduces new

arch/arm64/Makefile

+8 −1

Original line number	Diff line number	Diff line
		@@ -39,6 +39,12 @@ $(warning LSE atomics not supported by binutils)
		endif
		endif

		cc_has_k_constraint := $(call try-run,echo \
		'int main(void) { \
		asm volatile("and w0, w0, %w0" :: "K" (4294967295)); \
		return 0; \
		}' \| $(CC) -S -x c -o "$$TMP" -,,-DCONFIG_CC_HAS_K_CONSTRAINT=1)

		ifeq ($(CONFIG_ARM64), y)
		brokengasinst := $(call as-instr,1:\n.inst 0\n.rept . - 1b\n\nnop\n.endr\n,,-DCONFIG_BROKEN_GAS_INST=1)

		@@ -63,7 +69,8 @@ ifeq ($(CONFIG_GENERIC_COMPAT_VDSO), y)
		endif
		endif

		KBUILD_CFLAGS += -mgeneral-regs-only $(lseinstr) $(brokengasinst) $(compat_vdso)
		KBUILD_CFLAGS += -mgeneral-regs-only $(lseinstr) $(brokengasinst) \
		$(compat_vdso) $(cc_has_k_constraint)
		KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
		KBUILD_CFLAGS += $(call cc-disable-warning, psabi)
		KBUILD_AFLAGS += $(lseinstr) $(brokengasinst) $(compat_vdso)

arch/arm64/include/asm/atomic.h

+83 −11

Original line number	Diff line number	Diff line
		@@ -13,19 +13,91 @@
		#include <linux/types.h>

		#include <asm/barrier.h>
		#include <asm/cmpxchg.h>
		#include <asm/lse.h>

		#define __ARM64_IN_ATOMIC_IMPL

		#if defined(CONFIG_ARM64_LSE_ATOMICS) && defined(CONFIG_AS_LSE)
		#include <asm/atomic_lse.h>
		#else
		#include <asm/atomic_ll_sc.h>
		#endif

		#undef __ARM64_IN_ATOMIC_IMPL

		#include <asm/cmpxchg.h>
		#define ATOMIC_OP(op) \
		static inline void arch_##op(int i, atomic_t *v) \
		{ \
		__lse_ll_sc_body(op, i, v); \
		}

		ATOMIC_OP(atomic_andnot)
		ATOMIC_OP(atomic_or)
		ATOMIC_OP(atomic_xor)
		ATOMIC_OP(atomic_add)
		ATOMIC_OP(atomic_and)
		ATOMIC_OP(atomic_sub)

		#undef ATOMIC_OP

		#define ATOMIC_FETCH_OP(name, op) \
		static inline int arch_##op##name(int i, atomic_t *v) \
		{ \
		return __lse_ll_sc_body(op##name, i, v); \
		}

		#define ATOMIC_FETCH_OPS(op) \
		ATOMIC_FETCH_OP(_relaxed, op) \
		ATOMIC_FETCH_OP(_acquire, op) \
		ATOMIC_FETCH_OP(_release, op) \
		ATOMIC_FETCH_OP( , op)

		ATOMIC_FETCH_OPS(atomic_fetch_andnot)
		ATOMIC_FETCH_OPS(atomic_fetch_or)
		ATOMIC_FETCH_OPS(atomic_fetch_xor)
		ATOMIC_FETCH_OPS(atomic_fetch_add)
		ATOMIC_FETCH_OPS(atomic_fetch_and)
		ATOMIC_FETCH_OPS(atomic_fetch_sub)
		ATOMIC_FETCH_OPS(atomic_add_return)
		ATOMIC_FETCH_OPS(atomic_sub_return)

		#undef ATOMIC_FETCH_OP
		#undef ATOMIC_FETCH_OPS

		#define ATOMIC64_OP(op) \
		static inline void arch_##op(long i, atomic64_t *v) \
		{ \
		__lse_ll_sc_body(op, i, v); \
		}

		ATOMIC64_OP(atomic64_andnot)
		ATOMIC64_OP(atomic64_or)
		ATOMIC64_OP(atomic64_xor)
		ATOMIC64_OP(atomic64_add)
		ATOMIC64_OP(atomic64_and)
		ATOMIC64_OP(atomic64_sub)

		#undef ATOMIC64_OP

		#define ATOMIC64_FETCH_OP(name, op) \
		static inline long arch_##op##name(long i, atomic64_t *v) \
		{ \
		return __lse_ll_sc_body(op##name, i, v); \
		}

		#define ATOMIC64_FETCH_OPS(op) \
		ATOMIC64_FETCH_OP(_relaxed, op) \
		ATOMIC64_FETCH_OP(_acquire, op) \
		ATOMIC64_FETCH_OP(_release, op) \
		ATOMIC64_FETCH_OP( , op)

		ATOMIC64_FETCH_OPS(atomic64_fetch_andnot)
		ATOMIC64_FETCH_OPS(atomic64_fetch_or)
		ATOMIC64_FETCH_OPS(atomic64_fetch_xor)
		ATOMIC64_FETCH_OPS(atomic64_fetch_add)
		ATOMIC64_FETCH_OPS(atomic64_fetch_and)
		ATOMIC64_FETCH_OPS(atomic64_fetch_sub)
		ATOMIC64_FETCH_OPS(atomic64_add_return)
		ATOMIC64_FETCH_OPS(atomic64_sub_return)

		#undef ATOMIC64_FETCH_OP
		#undef ATOMIC64_FETCH_OPS

		static inline long arch_atomic64_dec_if_positive(atomic64_t *v)
		{
		return __lse_ll_sc_body(atomic64_dec_if_positive, v);
		}

		#define ATOMIC_INIT(i) { (i) }

arch/arm64/include/asm/atomic_ll_sc.h

+121 −96

Original line number	Diff line number	Diff line
		@@ -10,83 +10,92 @@
		#ifndef __ASM_ATOMIC_LL_SC_H
		#define __ASM_ATOMIC_LL_SC_H

		#ifndef __ARM64_IN_ATOMIC_IMPL
		#error "please don't include this file directly"
		#include <linux/stringify.h>

		#if IS_ENABLED(CONFIG_ARM64_LSE_ATOMICS) && IS_ENABLED(CONFIG_AS_LSE)
		#define __LL_SC_FALLBACK(asm_ops) \
		" b 3f\n" \
		" .subsection 1\n" \
		"3:\n" \
		asm_ops "\n" \
		" b 4f\n" \
		" .previous\n" \
		"4:\n"
		#else
		#define __LL_SC_FALLBACK(asm_ops) asm_ops
		#endif

		#ifndef CONFIG_CC_HAS_K_CONSTRAINT
		#define K
		#endif

		/*
		* AArch64 UP and SMP safe atomic ops. We use load exclusive and
		* store exclusive to ensure that these are atomic. We may loop
		* to ensure that the update happens.
		*
		* NOTE: these functions do not follow the PCS and must explicitly
		* save any clobbered registers other than x0 (regardless of return
		* value). This is achieved through -fcall-saved-* compiler flags for
		* this file, which unfortunately don't work on a per-function basis
		* (the optimize attribute silently ignores these options).
		*/

		#define ATOMIC_OP(op, asm_op) \
		__LL_SC_INLINE void \
		__LL_SC_PREFIX(arch_atomic_##op(int i, atomic_t *v)) \
		#define ATOMIC_OP(op, asm_op, constraint) \
		static inline void \
		__ll_sc_atomic_##op(int i, atomic_t *v) \
		{ \
		unsigned long tmp; \
		int result; \
		\
		asm volatile("// atomic_" #op "\n" \
		__LL_SC_FALLBACK( \
		" prfm pstl1strm, %2\n" \
		"1: ldxr %w0, %2\n" \
		" " #asm_op " %w0, %w0, %w3\n" \
		" stxr %w1, %w0, %2\n" \
		" cbnz %w1, 1b" \
		" cbnz %w1, 1b\n") \
		: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
		: "Ir" (i)); \
		} \
		__LL_SC_EXPORT(arch_atomic_##op);
		: __stringify(constraint) "r" (i)); \
		}

		#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
		__LL_SC_INLINE int \
		__LL_SC_PREFIX(arch_atomic_##op##_return##name(int i, atomic_t *v)) \
		#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
		static inline int \
		__ll_sc_atomic_##op##_return##name(int i, atomic_t *v) \
		{ \
		unsigned long tmp; \
		int result; \
		\
		asm volatile("// atomic_" #op "_return" #name "\n" \
		__LL_SC_FALLBACK( \
		" prfm pstl1strm, %2\n" \
		"1: ld" #acq "xr %w0, %2\n" \
		" " #asm_op " %w0, %w0, %w3\n" \
		" st" #rel "xr %w1, %w0, %2\n" \
		" cbnz %w1, 1b\n" \
		" " #mb \
		" " #mb ) \
		: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
		: "Ir" (i) \
		: __stringify(constraint) "r" (i) \
		: cl); \
		\
		return result; \
		} \
		__LL_SC_EXPORT(arch_atomic_##op##_return##name);
		}

		#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op) \
		__LL_SC_INLINE int \
		__LL_SC_PREFIX(arch_atomic_fetch_##op##name(int i, atomic_t *v)) \
		#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \
		static inline int \
		__ll_sc_atomic_fetch_##op##name(int i, atomic_t *v) \
		{ \
		unsigned long tmp; \
		int val, result; \
		\
		asm volatile("// atomic_fetch_" #op #name "\n" \
		__LL_SC_FALLBACK( \
		" prfm pstl1strm, %3\n" \
		"1: ld" #acq "xr %w0, %3\n" \
		" " #asm_op " %w1, %w0, %w4\n" \
		" st" #rel "xr %w2, %w1, %3\n" \
		" cbnz %w2, 1b\n" \
		" " #mb \
		" " #mb ) \
		: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \
		: "Ir" (i) \
		: __stringify(constraint) "r" (i) \
		: cl); \
		\
		return result; \
		} \
		__LL_SC_EXPORT(arch_atomic_fetch_##op##name);
		}

		#define ATOMIC_OPS(...) \
		ATOMIC_OP(__VA_ARGS__) \
		@@ -99,8 +108,8 @@ __LL_SC_EXPORT(arch_atomic_fetch_##op##name);
		ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\
		ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__)

		ATOMIC_OPS(add, add)
		ATOMIC_OPS(sub, sub)
		ATOMIC_OPS(add, add, I)
		ATOMIC_OPS(sub, sub, J)

		#undef ATOMIC_OPS
		#define ATOMIC_OPS(...) \
		@@ -110,77 +119,82 @@ ATOMIC_OPS(sub, sub)
		ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\
		ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__)

		ATOMIC_OPS(and, and)
		ATOMIC_OPS(andnot, bic)
		ATOMIC_OPS(or, orr)
		ATOMIC_OPS(xor, eor)
		ATOMIC_OPS(and, and, K)
		ATOMIC_OPS(or, orr, K)
		ATOMIC_OPS(xor, eor, K)
		/*
		* GAS converts the mysterious and undocumented BIC (immediate) alias to
		* an AND (immediate) instruction with the immediate inverted. We don't
		* have a constraint for this, so fall back to register.
		*/
		ATOMIC_OPS(andnot, bic, )

		#undef ATOMIC_OPS
		#undef ATOMIC_FETCH_OP
		#undef ATOMIC_OP_RETURN
		#undef ATOMIC_OP

		#define ATOMIC64_OP(op, asm_op) \
		__LL_SC_INLINE void \
		__LL_SC_PREFIX(arch_atomic64_##op(s64 i, atomic64_t *v)) \
		#define ATOMIC64_OP(op, asm_op, constraint) \
		static inline void \
		__ll_sc_atomic64_##op(s64 i, atomic64_t *v) \
		{ \
		s64 result; \
		unsigned long tmp; \
		\
		asm volatile("// atomic64_" #op "\n" \
		__LL_SC_FALLBACK( \
		" prfm pstl1strm, %2\n" \
		"1: ldxr %0, %2\n" \
		" " #asm_op " %0, %0, %3\n" \
		" stxr %w1, %0, %2\n" \
		" cbnz %w1, 1b" \
		" cbnz %w1, 1b") \
		: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
		: "Ir" (i)); \
		} \
		__LL_SC_EXPORT(arch_atomic64_##op);
		: __stringify(constraint) "r" (i)); \
		}

		#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
		__LL_SC_INLINE s64 \
		__LL_SC_PREFIX(arch_atomic64_##op##_return##name(s64 i, atomic64_t *v))\
		#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
		static inline long \
		__ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v) \
		{ \
		s64 result; \
		unsigned long tmp; \
		\
		asm volatile("// atomic64_" #op "_return" #name "\n" \
		__LL_SC_FALLBACK( \
		" prfm pstl1strm, %2\n" \
		"1: ld" #acq "xr %0, %2\n" \
		" " #asm_op " %0, %0, %3\n" \
		" st" #rel "xr %w1, %0, %2\n" \
		" cbnz %w1, 1b\n" \
		" " #mb \
		" " #mb ) \
		: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
		: "Ir" (i) \
		: __stringify(constraint) "r" (i) \
		: cl); \
		\
		return result; \
		} \
		__LL_SC_EXPORT(arch_atomic64_##op##_return##name);
		}

		#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op) \
		__LL_SC_INLINE s64 \
		__LL_SC_PREFIX(arch_atomic64_fetch_##op##name(s64 i, atomic64_t *v)) \
		#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\
		static inline long \
		__ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v) \
		{ \
		s64 result, val; \
		unsigned long tmp; \
		\
		asm volatile("// atomic64_fetch_" #op #name "\n" \
		__LL_SC_FALLBACK( \
		" prfm pstl1strm, %3\n" \
		"1: ld" #acq "xr %0, %3\n" \
		" " #asm_op " %1, %0, %4\n" \
		" st" #rel "xr %w2, %1, %3\n" \
		" cbnz %w2, 1b\n" \
		" " #mb \
		" " #mb ) \
		: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \
		: "Ir" (i) \
		: __stringify(constraint) "r" (i) \
		: cl); \
		\
		return result; \
		} \
		__LL_SC_EXPORT(arch_atomic64_fetch_##op##name);
		}

		#define ATOMIC64_OPS(...) \
		ATOMIC64_OP(__VA_ARGS__) \
		@@ -193,8 +207,8 @@ __LL_SC_EXPORT(arch_atomic64_fetch_##op##name);
		ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \
		ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__)

		ATOMIC64_OPS(add, add)
		ATOMIC64_OPS(sub, sub)
		ATOMIC64_OPS(add, add, I)
		ATOMIC64_OPS(sub, sub, J)

		#undef ATOMIC64_OPS
		#define ATOMIC64_OPS(...) \
		@@ -204,23 +218,29 @@ ATOMIC64_OPS(sub, sub)
		ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \
		ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__)

		ATOMIC64_OPS(and, and)
		ATOMIC64_OPS(andnot, bic)
		ATOMIC64_OPS(or, orr)
		ATOMIC64_OPS(xor, eor)
		ATOMIC64_OPS(and, and, L)
		ATOMIC64_OPS(or, orr, L)
		ATOMIC64_OPS(xor, eor, L)
		/*
		* GAS converts the mysterious and undocumented BIC (immediate) alias to
		* an AND (immediate) instruction with the immediate inverted. We don't
		* have a constraint for this, so fall back to register.
		*/
		ATOMIC64_OPS(andnot, bic, )

		#undef ATOMIC64_OPS
		#undef ATOMIC64_FETCH_OP
		#undef ATOMIC64_OP_RETURN
		#undef ATOMIC64_OP

		__LL_SC_INLINE s64
		__LL_SC_PREFIX(arch_atomic64_dec_if_positive(atomic64_t *v))
		static inline s64
		__ll_sc_atomic64_dec_if_positive(atomic64_t *v)
		{
		s64 result;
		unsigned long tmp;

		asm volatile("// atomic64_dec_if_positive\n"
		__LL_SC_FALLBACK(
		" prfm pstl1strm, %2\n"
		"1: ldxr %0, %2\n"
		" subs %0, %0, #1\n"
		@@ -228,20 +248,19 @@ __LL_SC_PREFIX(arch_atomic64_dec_if_positive(atomic64_t *v))
		" stlxr %w1, %0, %2\n"
		" cbnz %w1, 1b\n"
		" dmb ish\n"
		"2:"
		"2:")
		: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
		:
		: "cc", "memory");

		return result;
		}
		__LL_SC_EXPORT(arch_atomic64_dec_if_positive);

		#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl) \
		__LL_SC_INLINE u##sz \
		__LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \
		#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint) \
		static inline u##sz \
		__ll_sc__cmpxchg_case_##name##sz(volatile void *ptr, \
		unsigned long old, \
		u##sz new)) \
		u##sz new) \
		{ \
		unsigned long tmp; \
		u##sz oldval; \
		@@ -255,6 +274,7 @@ __LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \
		old = (u##sz)old; \
		\
		asm volatile( \
		__LL_SC_FALLBACK( \
		" prfm pstl1strm, %[v]\n" \
		"1: ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n" \
		" eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
		@@ -262,46 +282,51 @@ __LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \
		" st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \
		" cbnz %w[tmp], 1b\n" \
		" " #mb "\n" \
		"2:" \
		"2:") \
		: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \
		[v] "+Q" ((u##sz )ptr) \
		: [old] "Kr" (old), [new] "r" (new) \
		: [old] __stringify(constraint) "r" (old), [new] "r" (new) \
		: cl); \
		\
		return oldval; \
		} \
		__LL_SC_EXPORT(__cmpxchg_case_##name##sz);
		}

		__CMPXCHG_CASE(w, b, , 8, , , , )
		__CMPXCHG_CASE(w, h, , 16, , , , )
		__CMPXCHG_CASE(w, , , 32, , , , )
		__CMPXCHG_CASE( , , , 64, , , , )
		__CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory")
		__CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory")
		__CMPXCHG_CASE(w, , acq_, 32, , a, , "memory")
		__CMPXCHG_CASE( , , acq_, 64, , a, , "memory")
		__CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory")
		__CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory")
		__CMPXCHG_CASE(w, , rel_, 32, , , l, "memory")
		__CMPXCHG_CASE( , , rel_, 64, , , l, "memory")
		__CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory")
		__CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory")
		__CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory")
		__CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory")
		/*
		* Earlier versions of GCC (no later than 8.1.0) appear to incorrectly
		* handle the 'K' constraint for the value 4294967295 - thus we use no
		* constraint for 32 bit operations.
		*/
		__CMPXCHG_CASE(w, b, , 8, , , , , K)
		__CMPXCHG_CASE(w, h, , 16, , , , , K)
		__CMPXCHG_CASE(w, , , 32, , , , , K)
		__CMPXCHG_CASE( , , , 64, , , , , L)
		__CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K)
		__CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K)
		__CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K)
		__CMPXCHG_CASE( , , acq_, 64, , a, , "memory", L)
		__CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory", K)
		__CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K)
		__CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K)
		__CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L)
		__CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K)
		__CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K)
		__CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K)
		__CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L)

		#undef __CMPXCHG_CASE

		#define __CMPXCHG_DBL(name, mb, rel, cl) \
		__LL_SC_INLINE long \
		__LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \
		static inline long \
		__ll_sc__cmpxchg_double##name(unsigned long old1, \
		unsigned long old2, \
		unsigned long new1, \
		unsigned long new2, \
		volatile void *ptr)) \
		volatile void *ptr) \
		{ \
		unsigned long tmp, ret; \
		\
		asm volatile("// __cmpxchg_double" #name "\n" \
		__LL_SC_FALLBACK( \
		" prfm pstl1strm, %2\n" \
		"1: ldxp %0, %1, %2\n" \
		" eor %0, %0, %3\n" \
		@@ -311,18 +336,18 @@ __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \
		" st" #rel "xp %w0, %5, %6, %2\n" \
		" cbnz %w0, 1b\n" \
		" " #mb "\n" \
		"2:" \
		"2:") \
		: "=&r" (tmp), "=&r" (ret), "+Q" ((unsigned long )ptr) \
		: "r" (old1), "r" (old2), "r" (new1), "r" (new2) \
		: cl); \
		\
		return ret; \
		} \
		__LL_SC_EXPORT(__cmpxchg_double##name);
		}

		__CMPXCHG_DBL( , , , )
		__CMPXCHG_DBL(_mb, dmb ish, l, "memory")

		#undef __CMPXCHG_DBL
		#undef K

		#endif /* __ASM_ATOMIC_LL_SC_H */

arch/arm64/include/asm/atomic_lse.h

+135 −260

File changed.

Preview size limit exceeded, changes collapsed.