Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d6cd4715 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'x86-rwsem-for-linus' of... 

Merge branch 'x86-rwsem-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'x86-rwsem-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
  x86-64, rwsem: Avoid store forwarding hazard in __downgrade_write
  x86-64, rwsem: 64-bit xadd rwsem implementation
  x86: Fix breakage of UML from the changes in the rwsem system
  x86-64: support native xadd rwsem implementation
  x86: clean up rwsem type system
parents 1eca9acb 0d1622d7

arch/um/sys-x86_64/Makefile

+2 −1
Original line number Diff line number Diff line
@@ -8,7 +8,8 @@ obj-y = bug.o bugs.o delay.o fault.o ldt.o mem.o ptrace.o ptrace_user.o \ 
	setjmp.o signal.o stub.o stub_segv.o syscalls.o syscall_table.o \

	sysrq.o ksyms.o tls.o



subarch-obj-y = lib/csum-partial_64.o lib/memcpy_64.o lib/thunk_64.o

subarch-obj-y = lib/csum-partial_64.o lib/memcpy_64.o lib/thunk_64.o \

		lib/rwsem_64.o

subarch-obj-$(CONFIG_MODULES) += kernel/module.o



ldt-y = ../sys-i386/ldt.o

arch/x86/Kconfig.cpu

+1 −1
Original line number Diff line number Diff line
@@ -319,7 +319,7 @@ config X86_L1_CACHE_SHIFT 


config X86_XADD

	def_bool y

	depends on X86_32 && !M386

	depends on X86_64 || !M386



config X86_PPRO_FENCE

	bool "PentiumPro memory ordering errata workaround"

arch/x86/include/asm/rwsem.h

+38 −21
Original line number Diff line number Diff line
@@ -41,6 +41,7 @@ 
#include <linux/list.h>

#include <linux/spinlock.h>

#include <linux/lockdep.h>

#include <asm/asm.h>



struct rwsem_waiter;
@@ -55,17 +56,28 @@ extern asmregparm struct rw_semaphore * 


/*

 * the semaphore definition

 *

 * The bias values and the counter type limits the number of

 * potential readers/writers to 32767 for 32 bits and 2147483647

 * for 64 bits.

 */



#define RWSEM_UNLOCKED_VALUE		0x00000000

#define RWSEM_ACTIVE_BIAS		0x00000001

#define RWSEM_ACTIVE_MASK		0x0000ffff

#define RWSEM_WAITING_BIAS		(-0x00010000)

#ifdef CONFIG_X86_64

# define RWSEM_ACTIVE_MASK		0xffffffffL

#else

# define RWSEM_ACTIVE_MASK		0x0000ffffL

#endif



#define RWSEM_UNLOCKED_VALUE		0x00000000L

#define RWSEM_ACTIVE_BIAS		0x00000001L

#define RWSEM_WAITING_BIAS		(-RWSEM_ACTIVE_MASK-1)

#define RWSEM_ACTIVE_READ_BIAS		RWSEM_ACTIVE_BIAS

#define RWSEM_ACTIVE_WRITE_BIAS		(RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)



typedef signed long rwsem_count_t;



struct rw_semaphore {

	signed long		count;

	rwsem_count_t		count;

	spinlock_t		wait_lock;

	struct list_head	wait_list;

#ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -105,7 +117,7 @@ do { \ 
static inline void __down_read(struct rw_semaphore *sem)

{

	asm volatile("# beginning down_read\n\t"

		     LOCK_PREFIX "  inc%z0      (%1)\n\t"

		     LOCK_PREFIX _ASM_INC "(%1)\n\t"

		     /* adds 0x00000001, returns the old value */

		     "  jns        1f\n"

		     "  call call_rwsem_down_read_failed\n"
@@ -121,7 +133,7 @@ static inline void __down_read(struct rw_semaphore *sem) 
 */

static inline int __down_read_trylock(struct rw_semaphore *sem)

{

	__s32 result, tmp;

	rwsem_count_t result, tmp;

	asm volatile("# beginning __down_read_trylock\n\t"

		     "  mov          %0,%1\n\t"

		     "1:\n\t"
@@ -143,7 +155,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) 
 */

static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)

{

	int tmp;

	rwsem_count_t tmp;



	tmp = RWSEM_ACTIVE_WRITE_BIAS;

	asm volatile("# beginning down_write\n\t"
@@ -170,7 +182,7 @@ static inline void __down_write(struct rw_semaphore *sem) 
 */

static inline int __down_write_trylock(struct rw_semaphore *sem)

{

	signed long ret = cmpxchg(&sem->count,

	rwsem_count_t ret = cmpxchg(&sem->count,

				    RWSEM_UNLOCKED_VALUE,

				    RWSEM_ACTIVE_WRITE_BIAS);

	if (ret == RWSEM_UNLOCKED_VALUE)
@@ -183,7 +195,7 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) 
 */

static inline void __up_read(struct rw_semaphore *sem)

{

	__s32 tmp = -RWSEM_ACTIVE_READ_BIAS;

	rwsem_count_t tmp = -RWSEM_ACTIVE_READ_BIAS;

	asm volatile("# beginning __up_read\n\t"

		     LOCK_PREFIX "  xadd      %1,(%2)\n\t"

		     /* subtracts 1, returns the old value */
@@ -201,7 +213,7 @@ static inline void __up_read(struct rw_semaphore *sem) 
 */

static inline void __up_write(struct rw_semaphore *sem)

{

	unsigned long tmp;

	rwsem_count_t tmp;

	asm volatile("# beginning __up_write\n\t"

		     LOCK_PREFIX "  xadd      %1,(%2)\n\t"

		     /* tries to transition
@@ -221,33 +233,38 @@ static inline void __up_write(struct rw_semaphore *sem) 
static inline void __downgrade_write(struct rw_semaphore *sem)

{

	asm volatile("# beginning __downgrade_write\n\t"

		     LOCK_PREFIX "  add%z0    %2,(%1)\n\t"

		     /* transitions 0xZZZZ0001 -> 0xYYYY0001 */

		     LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t"

		     /*

		      * transitions 0xZZZZ0001 -> 0xYYYY0001 (i386)

		      *     0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64)

		      */

		     "  jns       1f\n\t"

		     "  call call_rwsem_downgrade_wake\n"

		     "1:\n\t"

		     "# ending __downgrade_write\n"

		     : "+m" (sem->count)

		     : "a" (sem), "i" (-RWSEM_WAITING_BIAS)

		     : "a" (sem), "er" (-RWSEM_WAITING_BIAS)

		     : "memory", "cc");

}



/*

 * implement atomic add functionality

 */

static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)

static inline void rwsem_atomic_add(rwsem_count_t delta,

				    struct rw_semaphore *sem)

{

	asm volatile(LOCK_PREFIX "add%z0 %1,%0"

	asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0"

		     : "+m" (sem->count)

		     : "ir" (delta));

		     : "er" (delta));

}



/*

 * implement exchange and add functionality

 */

static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)

static inline rwsem_count_t rwsem_atomic_update(rwsem_count_t delta,

						struct rw_semaphore *sem)

{

	int tmp = delta;

	rwsem_count_t tmp = delta;



	asm volatile(LOCK_PREFIX "xadd %0,%1"

		     : "+r" (tmp), "+m" (sem->count)

arch/x86/lib/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -39,4 +39,5 @@ else 
        lib-y += thunk_64.o clear_page_64.o copy_page_64.o

        lib-y += memmove_64.o memset_64.o

        lib-y += copy_user_64.o rwlock_64.o copy_user_nocache_64.o

	lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem_64.o

endif

arch/x86/lib/rwsem_64.S

0 → 100644
+81 −0
Original line number Diff line number Diff line 
/*

 * x86-64 rwsem wrappers

 *

 * This interfaces the inline asm code to the slow-path

 * C routines. We need to save the call-clobbered regs

 * that the asm does not mark as clobbered, and move the

 * argument from %rax to %rdi.

 *

 * NOTE! We don't need to save %rax, because the functions

 * will always return the semaphore pointer in %rax (which

 * is also the input argument to these helpers)

 *

 * The following can clobber %rdx because the asm clobbers it:

 *   call_rwsem_down_write_failed

 *   call_rwsem_wake

 * but %rdi, %rsi, %rcx, %r8-r11 always need saving.

 */



#include <linux/linkage.h>

#include <asm/rwlock.h>

#include <asm/alternative-asm.h>

#include <asm/frame.h>

#include <asm/dwarf2.h>



#define save_common_regs \

	pushq %rdi; \

	pushq %rsi; \

	pushq %rcx; \

	pushq %r8; \

	pushq %r9; \

	pushq %r10; \

	pushq %r11



#define restore_common_regs \

	popq %r11; \

	popq %r10; \

	popq %r9; \

	popq %r8; \

	popq %rcx; \

	popq %rsi; \

	popq %rdi



/* Fix up special calling conventions */

ENTRY(call_rwsem_down_read_failed)

	save_common_regs

	pushq %rdx

	movq %rax,%rdi

	call rwsem_down_read_failed

	popq %rdx

	restore_common_regs

	ret

	ENDPROC(call_rwsem_down_read_failed)



ENTRY(call_rwsem_down_write_failed)

	save_common_regs

	movq %rax,%rdi

	call rwsem_down_write_failed

	restore_common_regs

	ret

	ENDPROC(call_rwsem_down_write_failed)



ENTRY(call_rwsem_wake)

	decw %dx    /* do nothing if still outstanding active readers */

	jnz 1f

	save_common_regs

	movq %rax,%rdi

	call rwsem_wake

	restore_common_regs

1:	ret

	ENDPROC(call_rwsem_wake)



/* Fix up special calling conventions */

ENTRY(call_rwsem_downgrade_wake)

	save_common_regs

	pushq %rdx

	movq %rax,%rdi

	call rwsem_downgrade_wake

	popq %rdx

	restore_common_regs

	ret

	ENDPROC(call_rwsem_downgrade_wake)