Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 69734b64 authored Jan 06, 2012 by Linus Torvalds

Merge branch 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

* 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (21 commits)
  x86: Fix atomic64_xxx_cx8() functions
  x86: Fix and improve cmpxchg_double{,_local}()
  x86_64, asm: Optimise fls(), ffs() and fls64()
  x86, bitops: Move fls64.h inside __KERNEL__
  x86: Fix and improve percpu_cmpxchg{8,16}b_double()
  x86: Report cpb and eff_freq_ro flags correctly
  x86/i386: Use less assembly in strlen(), speed things up a bit
  x86: Use the same node_distance for 32 and 64-bit
  x86: Fix rflags in FAKE_STACK_FRAME
  x86: Clean up and extend do_int3()
  x86: Call do_notify_resume() with interrupts enabled
  x86/div64: Add a micro-optimization shortcut if base is power of two
  x86-64: Cleanup some assembly entry points
  x86-64: Slightly shorten line system call entry and exit paths
  x86-64: Reduce amount of redundant code generated for invalidate_interruptNN
  x86-64: Slightly shorten int_ret_from_sys_call
  x86, efi: Convert efi_phys_get_time() args to physical addresses
  x86: Default to vsyscall=emulate
  x86-64: Set siginfo and context on vsyscall emulation faults
  x86: consolidate xchg and xadd macros
  ...

parents 67b02431 ceb7b40b

Documentation/kernel-parameters.txt

+3 −4

Original line number	Original line	Diff line number	Diff line
	@@ -2755,11 +2755,10 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
	functions are at fixed addresses, they make nice		functions are at fixed addresses, they make nice
	targets for exploits that can control RIP.		targets for exploits that can control RIP.

	emulate Vsyscalls turn into traps and are emulated		emulate [default] Vsyscalls turn into traps and are
	reasonably safely.		emulated reasonably safely.

	native [default] Vsyscalls are native syscall		native Vsyscalls are native syscall instructions.
	instructions.
	This is a little bit faster than trapping		This is a little bit faster than trapping
	and makes a few dynamic recompilers work		and makes a few dynamic recompilers work
	better than they would in emulation mode.		better than they would in emulation mode.

arch/x86/ia32/ia32entry.S

+19 −24

Original line number	Original line	Diff line number	Diff line
	@@ -134,7 +134,7 @@ ENTRY(ia32_sysenter_target)
	CFI_REL_OFFSET rsp,0		CFI_REL_OFFSET rsp,0
	pushfq_cfi		pushfq_cfi
	/CFI_REL_OFFSET rflags,0/		/CFI_REL_OFFSET rflags,0/
	movl 8*3-THREAD_SIZE+TI_sysenter_return(%rsp), %r10d		movl TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d
	CFI_REGISTER rip,r10		CFI_REGISTER rip,r10
	pushq_cfi $__USER32_CS		pushq_cfi $__USER32_CS
	/CFI_REL_OFFSET cs,0/		/CFI_REL_OFFSET cs,0/
	@@ -150,9 +150,8 @@ ENTRY(ia32_sysenter_target)
	.section __ex_table,"a"		.section __ex_table,"a"
	.quad 1b,ia32_badarg		.quad 1b,ia32_badarg
	.previous		.previous
	GET_THREAD_INFO(%r10)		orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	orl $TS_COMPAT,TI_status(%r10)		testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)
	CFI_REMEMBER_STATE		CFI_REMEMBER_STATE
	jnz sysenter_tracesys		jnz sysenter_tracesys
	cmpq $(IA32_NR_syscalls-1),%rax		cmpq $(IA32_NR_syscalls-1),%rax
	@@ -162,13 +161,12 @@ sysenter_do_call:
	sysenter_dispatch:		sysenter_dispatch:
	call *ia32_sys_call_table(,%rax,8)		call *ia32_sys_call_table(,%rax,8)
	movq %rax,RAX-ARGOFFSET(%rsp)		movq %rax,RAX-ARGOFFSET(%rsp)
	GET_THREAD_INFO(%r10)
	DISABLE_INTERRUPTS(CLBR_NONE)		DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF		TRACE_IRQS_OFF
	testl $_TIF_ALLWORK_MASK,TI_flags(%r10)		testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	jnz sysexit_audit		jnz sysexit_audit
	sysexit_from_sys_call:		sysexit_from_sys_call:
	andl $~TS_COMPAT,TI_status(%r10)		andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	/* clear IF, that popfq doesn't enable interrupts early */		/* clear IF, that popfq doesn't enable interrupts early */
	andl $~0x200,EFLAGS-R11(%rsp)		andl $~0x200,EFLAGS-R11(%rsp)
	movl RIP-R11(%rsp),%edx /* User %eip */		movl RIP-R11(%rsp),%edx /* User %eip */
	@@ -205,7 +203,7 @@ sysexit_from_sys_call:
	.endm		.endm

	.macro auditsys_exit exit		.macro auditsys_exit exit
	testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)		testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	jnz ia32_ret_from_sys_call		jnz ia32_ret_from_sys_call
	TRACE_IRQS_ON		TRACE_IRQS_ON
	sti		sti
	@@ -215,12 +213,11 @@ sysexit_from_sys_call:
	movzbl %al,%edi /* zero-extend that into %edi */		movzbl %al,%edi /* zero-extend that into %edi */
	inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */		inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
	call audit_syscall_exit		call audit_syscall_exit
	GET_THREAD_INFO(%r10)
	movl RAX-ARGOFFSET(%rsp),%eax /* reload syscall return value */		movl RAX-ARGOFFSET(%rsp),%eax /* reload syscall return value */
	movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi		movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
	cli		cli
	TRACE_IRQS_OFF		TRACE_IRQS_OFF
	testl %edi,TI_flags(%r10)		testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	jz \exit		jz \exit
	CLEAR_RREGS -ARGOFFSET		CLEAR_RREGS -ARGOFFSET
	jmp int_with_check		jmp int_with_check
	@@ -238,7 +235,7 @@ sysexit_audit:

	sysenter_tracesys:		sysenter_tracesys:
	#ifdef CONFIG_AUDITSYSCALL		#ifdef CONFIG_AUDITSYSCALL
	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)		testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	jz sysenter_auditsys		jz sysenter_auditsys
	#endif		#endif
	SAVE_REST		SAVE_REST
	@@ -309,9 +306,8 @@ ENTRY(ia32_cstar_target)
	.section __ex_table,"a"		.section __ex_table,"a"
	.quad 1b,ia32_badarg		.quad 1b,ia32_badarg
	.previous		.previous
	GET_THREAD_INFO(%r10)		orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	orl $TS_COMPAT,TI_status(%r10)		testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)
	CFI_REMEMBER_STATE		CFI_REMEMBER_STATE
	jnz cstar_tracesys		jnz cstar_tracesys
	cmpq $IA32_NR_syscalls-1,%rax		cmpq $IA32_NR_syscalls-1,%rax
	@@ -321,13 +317,12 @@ cstar_do_call:
	cstar_dispatch:		cstar_dispatch:
	call *ia32_sys_call_table(,%rax,8)		call *ia32_sys_call_table(,%rax,8)
	movq %rax,RAX-ARGOFFSET(%rsp)		movq %rax,RAX-ARGOFFSET(%rsp)
	GET_THREAD_INFO(%r10)
	DISABLE_INTERRUPTS(CLBR_NONE)		DISABLE_INTERRUPTS(CLBR_NONE)
	TRACE_IRQS_OFF		TRACE_IRQS_OFF
	testl $_TIF_ALLWORK_MASK,TI_flags(%r10)		testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	jnz sysretl_audit		jnz sysretl_audit
	sysretl_from_sys_call:		sysretl_from_sys_call:
	andl $~TS_COMPAT,TI_status(%r10)		andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	RESTORE_ARGS 0,-ARG_SKIP,0,0,0		RESTORE_ARGS 0,-ARG_SKIP,0,0,0
	movl RIP-ARGOFFSET(%rsp),%ecx		movl RIP-ARGOFFSET(%rsp),%ecx
	CFI_REGISTER rip,rcx		CFI_REGISTER rip,rcx
	@@ -355,7 +350,7 @@ sysretl_audit:

	cstar_tracesys:		cstar_tracesys:
	#ifdef CONFIG_AUDITSYSCALL		#ifdef CONFIG_AUDITSYSCALL
	testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%r10)		testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	jz cstar_auditsys		jz cstar_auditsys
	#endif		#endif
	xchgl %r9d,%ebp		xchgl %r9d,%ebp
	@@ -420,9 +415,8 @@ ENTRY(ia32_syscall)
	/* note the registers are not zero extended to the sf.		/* note the registers are not zero extended to the sf.
	this could be a problem. */		this could be a problem. */
	SAVE_ARGS 0,1,0		SAVE_ARGS 0,1,0
	GET_THREAD_INFO(%r10)		orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	orl $TS_COMPAT,TI_status(%r10)		testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
	testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)
	jnz ia32_tracesys		jnz ia32_tracesys
	cmpq $(IA32_NR_syscalls-1),%rax		cmpq $(IA32_NR_syscalls-1),%rax
	ja ia32_badsys		ja ia32_badsys
	@@ -459,8 +453,8 @@ quiet_ni_syscall:
	CFI_ENDPROC		CFI_ENDPROC

	.macro PTREGSCALL label, func, arg		.macro PTREGSCALL label, func, arg
	.globl \label		ALIGN
	\label:		GLOBAL(\label)
	leaq \func(%rip),%rax		leaq \func(%rip),%rax
	leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */		leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
	jmp ia32_ptregs_common		jmp ia32_ptregs_common
	@@ -477,7 +471,8 @@ quiet_ni_syscall:
	PTREGSCALL stub32_vfork, sys_vfork, %rdi		PTREGSCALL stub32_vfork, sys_vfork, %rdi
	PTREGSCALL stub32_iopl, sys_iopl, %rsi		PTREGSCALL stub32_iopl, sys_iopl, %rsi

	ENTRY(ia32_ptregs_common)		ALIGN
			ia32_ptregs_common:
	popq %r11		popq %r11
	CFI_ENDPROC		CFI_ENDPROC
	CFI_STARTPROC32 simple		CFI_STARTPROC32 simple

arch/x86/include/asm/alternative-asm.h

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -4,10 +4,10 @@

	#ifdef CONFIG_SMP		#ifdef CONFIG_SMP
	.macro LOCK_PREFIX		.macro LOCK_PREFIX
	1: lock		672: lock
	.section .smp_locks,"a"		.section .smp_locks,"a"
	.balign 4		.balign 4
	.long 1b - .		.long 672b - .
	.previous		.previous
	.endm		.endm
	#else		#else

arch/x86/include/asm/bitops.h

+63 −13

Original line number	Original line	Diff line number	Diff line
	@@ -380,6 +380,8 @@ static inline unsigned long __fls(unsigned long word)
	return word;		return word;
	}		}

			#undef ADDR

	#ifdef __KERNEL__		#ifdef __KERNEL__
	/**		/**
	* ffs - find first set bit in word		* ffs - find first set bit in word
	@@ -395,10 +397,25 @@ static inline unsigned long __fls(unsigned long word)
	static inline int ffs(int x)		static inline int ffs(int x)
	{		{
	int r;		int r;
	#ifdef CONFIG_X86_CMOV
			#ifdef CONFIG_X86_64
			/*
			* AMD64 says BSFL won't clobber the dest reg if x==0; Intel64 says the
			* dest reg is undefined if x==0, but their CPU architect says its
			* value is written to set it to the same as before, except that the
			* top 32 bits will be cleared.
			*
			* We cannot do this on 32 bits because at the very least some
			* 486 CPUs did not behave this way.
			*/
			long tmp = -1;
			asm("bsfl %1,%0"
			: "=r" (r)
			: "rm" (x), "0" (tmp));
			#elif defined(CONFIG_X86_CMOV)
	asm("bsfl %1,%0\n\t"		asm("bsfl %1,%0\n\t"
	"cmovzl %2,%0"		"cmovzl %2,%0"
	: "=r" (r) : "rm" (x), "r" (-1));		: "=&r" (r) : "rm" (x), "r" (-1));
	#else		#else
	asm("bsfl %1,%0\n\t"		asm("bsfl %1,%0\n\t"
	"jnz 1f\n\t"		"jnz 1f\n\t"
	@@ -422,7 +439,22 @@ static inline int ffs(int x)
	static inline int fls(int x)		static inline int fls(int x)
	{		{
	int r;		int r;
	#ifdef CONFIG_X86_CMOV
			#ifdef CONFIG_X86_64
			/*
			* AMD64 says BSRL won't clobber the dest reg if x==0; Intel64 says the
			* dest reg is undefined if x==0, but their CPU architect says its
			* value is written to set it to the same as before, except that the
			* top 32 bits will be cleared.
			*
			* We cannot do this on 32 bits because at the very least some
			* 486 CPUs did not behave this way.
			*/
			long tmp = -1;
			asm("bsrl %1,%0"
			: "=r" (r)
			: "rm" (x), "0" (tmp));
			#elif defined(CONFIG_X86_CMOV)
	asm("bsrl %1,%0\n\t"		asm("bsrl %1,%0\n\t"
	"cmovzl %2,%0"		"cmovzl %2,%0"
	: "=&r" (r) : "rm" (x), "rm" (-1));		: "=&r" (r) : "rm" (x), "rm" (-1));
	@@ -434,11 +466,35 @@ static inline int fls(int x)
	#endif		#endif
	return r + 1;		return r + 1;
	}		}
	#endif /* __KERNEL__ */

	#undef ADDR

	#ifdef __KERNEL__		/**
			* fls64 - find last set bit in a 64-bit word
			* @x: the word to search
			*
			* This is defined in a similar way as the libc and compiler builtin
			* ffsll, but returns the position of the most significant set bit.
			*
			* fls64(value) returns 0 if value is 0 or the position of the last
			* set bit if value is nonzero. The last (most significant) bit is
			* at position 64.
			*/
			#ifdef CONFIG_X86_64
			static __always_inline int fls64(__u64 x)
			{
			long bitpos = -1;
			/*
			* AMD64 says BSRQ won't clobber the dest reg if x==0; Intel64 says the
			* dest reg is undefined if x==0, but their CPU architect says its
			* value is written to set it to the same as before.
			*/
			asm("bsrq %1,%0"
			: "+r" (bitpos)
			: "rm" (x));
			return bitpos + 1;
			}
			#else
			#include <asm-generic/bitops/fls64.h>
			#endif

	#include <asm-generic/bitops/find.h>		#include <asm-generic/bitops/find.h>

	@@ -450,12 +506,6 @@ static inline int fls(int x)

	#include <asm-generic/bitops/const_hweight.h>		#include <asm-generic/bitops/const_hweight.h>

	#endif /* __KERNEL__ */

	#include <asm-generic/bitops/fls64.h>

	#ifdef __KERNEL__

	#include <asm-generic/bitops/le.h>		#include <asm-generic/bitops/le.h>

	#include <asm-generic/bitops/ext2-atomic-setbit.h>		#include <asm-generic/bitops/ext2-atomic-setbit.h>

arch/x86/include/asm/cmpxchg.h

+93 −70

Original line number	Original line	Diff line number	Diff line
	@@ -14,6 +14,8 @@ extern void __cmpxchg_wrong_size(void)
	__compiletime_error("Bad argument size for cmpxchg");		__compiletime_error("Bad argument size for cmpxchg");
	extern void __xadd_wrong_size(void)		extern void __xadd_wrong_size(void)
	__compiletime_error("Bad argument size for xadd");		__compiletime_error("Bad argument size for xadd");
			extern void __add_wrong_size(void)
			__compiletime_error("Bad argument size for add");

	/*		/*
	* Constants for operation sizes. On 32-bit, the 64-bit size it set to		* Constants for operation sizes. On 32-bit, the 64-bit size it set to
	@@ -32,59 +34,46 @@ extern void __xadd_wrong_size(void)
	#endif		#endif

	/*		/*
	* Note: no "lock" prefix even on SMP: xchg always implies lock anyway.		* An exchange-type operation, which takes a value and a pointer, and
	* Since this is generally used to protect other memory information, we		* returns a the old value.
	* use "asm volatile" and "memory" clobbers to prevent gcc from moving
	* information around.
	*/		*/
	#define __xchg(x, ptr, size) \		#define __xchg_op(ptr, arg, op, lock) \
	({ \		({ \
	__typeof(*(ptr)) __x = (x); \		__typeof__ (*(ptr)) __ret = (arg); \
	switch (size) { \		switch (sizeof(*(ptr))) { \
	case __X86_CASE_B: \		case __X86_CASE_B: \
	{ \		asm volatile (lock #op "b %b0, %1\n" \
	volatile u8 __ptr = (volatile u8 )(ptr); \		: "+r" (__ret), "+m" (*(ptr)) \
	asm volatile("xchgb %0,%1" \		: : "memory", "cc"); \
	: "=q" (__x), "+m" (*__ptr) \
	: "0" (__x) \
	: "memory"); \
	break; \		break; \
	} \
	case __X86_CASE_W: \		case __X86_CASE_W: \
	{ \		asm volatile (lock #op "w %w0, %1\n" \
	volatile u16 __ptr = (volatile u16 )(ptr); \		: "+r" (__ret), "+m" (*(ptr)) \
	asm volatile("xchgw %0,%1" \		: : "memory", "cc"); \
	: "=r" (__x), "+m" (*__ptr) \
	: "0" (__x) \
	: "memory"); \
	break; \		break; \
	} \
	case __X86_CASE_L: \		case __X86_CASE_L: \
	{ \		asm volatile (lock #op "l %0, %1\n" \
	volatile u32 __ptr = (volatile u32 )(ptr); \		: "+r" (__ret), "+m" (*(ptr)) \
	asm volatile("xchgl %0,%1" \		: : "memory", "cc"); \
	: "=r" (__x), "+m" (*__ptr) \
	: "0" (__x) \
	: "memory"); \
	break; \		break; \
	} \
	case __X86_CASE_Q: \		case __X86_CASE_Q: \
	{ \		asm volatile (lock #op "q %q0, %1\n" \
	volatile u64 __ptr = (volatile u64 )(ptr); \		: "+r" (__ret), "+m" (*(ptr)) \
	asm volatile("xchgq %0,%1" \		: : "memory", "cc"); \
	: "=r" (__x), "+m" (*__ptr) \
	: "0" (__x) \
	: "memory"); \
	break; \		break; \
	} \
	default: \		default: \
	__xchg_wrong_size(); \		__ ## op ## _wrong_size(); \
	} \		} \
	__x; \		__ret; \
	})		})

	#define xchg(ptr, v) \		/*
	__xchg((v), (ptr), sizeof(*ptr))		* Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
			* Since this is generally used to protect other memory information, we
			* use "asm volatile" and "memory" clobbers to prevent gcc from moving
			* information around.
			*/
			#define xchg(ptr, v) __xchg_op((ptr), (v), xchg, "")

	/*		/*
	* Atomic compare and exchange. Compare OLD with MEM, if identical,		* Atomic compare and exchange. Compare OLD with MEM, if identical,
	@@ -165,46 +154,80 @@ extern void __xadd_wrong_size(void)
	__cmpxchg_local((ptr), (old), (new), sizeof(*ptr))		__cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
	#endif		#endif

	#define __xadd(ptr, inc, lock) \		/*
			* xadd() adds "inc" to "*ptr" and atomically returns the previous
			* value of "*ptr".
			*
			* xadd() is locked when multiple CPUs are online
			* xadd_sync() is always locked
			* xadd_local() is never locked
			*/
			#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock)
			#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)
			#define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ")
			#define xadd_local(ptr, inc) __xadd((ptr), (inc), "")

			#define __add(ptr, inc, lock) \
	({ \		({ \
	__typeof__ (*(ptr)) __ret = (inc); \		__typeof__ (*(ptr)) __ret = (inc); \
	switch (sizeof(*(ptr))) { \		switch (sizeof(*(ptr))) { \
	case __X86_CASE_B: \		case __X86_CASE_B: \
	asm volatile (lock "xaddb %b0, %1\n" \		asm volatile (lock "addb %b1, %0\n" \
	: "+r" (__ret), "+m" (*(ptr)) \		: "+m" (*(ptr)) : "ri" (inc) \
	: : "memory", "cc"); \		: "memory", "cc"); \
	break; \		break; \
	case __X86_CASE_W: \		case __X86_CASE_W: \
	asm volatile (lock "xaddw %w0, %1\n" \		asm volatile (lock "addw %w1, %0\n" \
	: "+r" (__ret), "+m" (*(ptr)) \		: "+m" (*(ptr)) : "ri" (inc) \
	: : "memory", "cc"); \		: "memory", "cc"); \
	break; \		break; \
	case __X86_CASE_L: \		case __X86_CASE_L: \
	asm volatile (lock "xaddl %0, %1\n" \		asm volatile (lock "addl %1, %0\n" \
	: "+r" (__ret), "+m" (*(ptr)) \		: "+m" (*(ptr)) : "ri" (inc) \
	: : "memory", "cc"); \		: "memory", "cc"); \
	break; \		break; \
	case __X86_CASE_Q: \		case __X86_CASE_Q: \
	asm volatile (lock "xaddq %q0, %1\n" \		asm volatile (lock "addq %1, %0\n" \
	: "+r" (__ret), "+m" (*(ptr)) \		: "+m" (*(ptr)) : "ri" (inc) \
	: : "memory", "cc"); \		: "memory", "cc"); \
	break; \		break; \
	default: \		default: \
	__xadd_wrong_size(); \		__add_wrong_size(); \
	} \		} \
	__ret; \		__ret; \
	})		})

	/*		/*
	* xadd() adds "inc" to "*ptr" and atomically returns the previous		* add_() adds "inc" to "ptr"
	* value of "*ptr".
	*		*
	* xadd() is locked when multiple CPUs are online		* __add() takes a lock prefix
	* xadd_sync() is always locked		* add_smp() is locked when multiple CPUs are online
	* xadd_local() is never locked		* add_sync() is always locked
	*/		*/
	#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)		#define add_smp(ptr, inc) __add((ptr), (inc), LOCK_PREFIX)
	#define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ")		#define add_sync(ptr, inc) __add((ptr), (inc), "lock; ")
	#define xadd_local(ptr, inc) __xadd((ptr), (inc), "")
			#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2) \
			({ \
			bool __ret; \
			__typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \
			__typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \
			BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \
			BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \
			VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \
			VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \
			asm volatile(pfx "cmpxchg%c4b %2; sete %0" \
			: "=a" (__ret), "+d" (__old2), \
			"+m" ((p1)), "+m" ((p2)) \
			: "i" (2 * sizeof(long)), "a" (__old1), \
			"b" (__new1), "c" (__new2)); \
			__ret; \
			})

			#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \
			__cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2)

			#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
			__cmpxchg_double(, p1, p2, o1, o2, n1, n2)

	#endif /* ASM_X86_CMPXCHG_H */		#endif /* ASM_X86_CMPXCHG_H */