Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4625cd63 authored by Jan Beulich's avatar Jan Beulich Committed by Ingo Molnar
Browse files

x86: Unify rwlock assembly implementation 



Rather than having two functionally identical implementations
for 32- and 64-bit configurations, extend the existing assembly
abstractions enough to fold the two rwlock implementations into
a shared one.

Signed-off-by: default avatarJan Beulich <jbeulich@novell.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/4E258DD7020000780004E3EA@nat28.tlf.novell.com


Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent dd2897bf

arch/x86/include/asm/asm.h

+2 −1
Original line number Diff line number Diff line
@@ -15,7 +15,8 @@ 
# define __ASM_SEL(a,b) __ASM_FORM(b)

#endif



#define __ASM_SIZE(inst)	__ASM_SEL(inst##l, inst##q)

#define __ASM_SIZE(inst, ...)	__ASM_SEL(inst##l##__VA_ARGS__, \

					  inst##q##__VA_ARGS__)

#define __ASM_REG(reg)		__ASM_SEL(e##reg, r##reg)



#define _ASM_PTR	__ASM_SEL(.long, .quad)

arch/x86/include/asm/frame.h

+6 −5
Original line number Diff line number Diff line 
#ifdef __ASSEMBLY__



#include <asm/asm.h>

#include <asm/dwarf2.h>



/* The annotation hides the frame from the unwinder and makes it look
@@ -7,13 +8,13 @@ 
   frame pointer later */

#ifdef CONFIG_FRAME_POINTER

	.macro FRAME

	pushl_cfi %ebp

	CFI_REL_OFFSET ebp,0

	movl %esp,%ebp

	__ASM_SIZE(push,_cfi)	%__ASM_REG(bp)

	CFI_REL_OFFSET		__ASM_REG(bp), 0

	__ASM_SIZE(mov)		%__ASM_REG(sp), %__ASM_REG(bp)

	.endm

	.macro ENDFRAME

	popl_cfi %ebp

	CFI_RESTORE ebp

	__ASM_SIZE(pop,_cfi)	%__ASM_REG(bp)

	CFI_RESTORE		__ASM_REG(bp)

	.endm

#else

	.macro FRAME

arch/x86/lib/Makefile

+4 −2
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ lib-y := delay.o 
lib-y += thunk_$(BITS).o

lib-y += usercopy_$(BITS).o getuser.o putuser.o

lib-y += memcpy_$(BITS).o

lib-$(CONFIG_SMP) += rwlock.o

lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o



obj-y += msr.o msr-reg.o msr-reg-export.o
@@ -29,7 +30,8 @@ ifeq ($(CONFIG_X86_32),y) 
        lib-y += atomic64_cx8_32.o

        lib-y += checksum_32.o

        lib-y += strstr_32.o

        lib-y += semaphore_32.o string_32.o

        lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += semaphore_32.o

        lib-y += string_32.o

        lib-y += cmpxchg.o

ifneq ($(CONFIG_X86_CMPXCHG64),y)

        lib-y += cmpxchg8b_emu.o atomic64_386_32.o
@@ -40,7 +42,7 @@ else 
        lib-y += csum-partial_64.o csum-copy_64.o csum-wrappers_64.o

        lib-y += thunk_64.o clear_page_64.o copy_page_64.o

        lib-y += memmove_64.o memset_64.o

        lib-y += copy_user_64.o rwlock_64.o copy_user_nocache_64.o

        lib-y += copy_user_64.o copy_user_nocache_64.o

	lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem_64.o

	lib-y += cmpxchg16b_emu.o

endif

arch/x86/lib/rwlock_64.S→arch/x86/lib/rwlock.S

+44 −0
Original line number Diff line number Diff line 
/* Slow paths of read/write spinlocks. */



#include <linux/linkage.h>

#include <asm/rwlock.h>

#include <asm/alternative-asm.h>

#include <asm/dwarf2.h>

#include <asm/frame.h>

#include <asm/rwlock.h>



#ifdef CONFIG_X86_32

# define __lock_ptr eax

#else

# define __lock_ptr rdi

#endif



/* rdi:	pointer to rwlock_t */

ENTRY(__write_lock_failed)

	CFI_STARTPROC

	LOCK_PREFIX

	addl $RW_LOCK_BIAS,(%rdi)

1:	rep

	nop

	cmpl $RW_LOCK_BIAS,(%rdi)

	FRAME

0:	LOCK_PREFIX

	addl	$RW_LOCK_BIAS, (%__lock_ptr)

1:	rep; nop

	cmpl	$RW_LOCK_BIAS, (%__lock_ptr)

	jne	1b

	LOCK_PREFIX

	subl $RW_LOCK_BIAS,(%rdi)

	jnz  __write_lock_failed

	subl	$RW_LOCK_BIAS, (%__lock_ptr)

	jnz	0b

	ENDFRAME

	ret

	CFI_ENDPROC

END(__write_lock_failed)



/* rdi:	pointer to rwlock_t */

ENTRY(__read_lock_failed)

	CFI_STARTPROC

	LOCK_PREFIX

	incl (%rdi)

1:	rep

	nop

	cmpl $1,(%rdi)

	FRAME

0:	LOCK_PREFIX

	incl	(%__lock_ptr)

1:	rep; nop

	cmpl	$1, (%__lock_ptr)

	js	1b

	LOCK_PREFIX

	decl (%rdi)

	js __read_lock_failed

	decl	(%__lock_ptr)

	js	0b

	ENDFRAME

	ret

	CFI_ENDPROC

END(__read_lock_failed)

arch/x86/lib/semaphore_32.S

+0 −44
Original line number Diff line number Diff line
@@ -14,8 +14,6 @@ 
 */



#include <linux/linkage.h>

#include <asm/rwlock.h>

#include <asm/alternative-asm.h>

#include <asm/frame.h>

#include <asm/dwarf2.h>
@@ -31,46 +29,6 @@ 
 */

	.section .sched.text, "ax"



/*

 * rw spinlock fallbacks

 */

#ifdef CONFIG_SMP

ENTRY(__write_lock_failed)

	CFI_STARTPROC

	FRAME

2: 	LOCK_PREFIX

	addl	$ RW_LOCK_BIAS,(%eax)

1:	rep; nop

	cmpl	$ RW_LOCK_BIAS,(%eax)

	jne	1b

	LOCK_PREFIX

	subl	$ RW_LOCK_BIAS,(%eax)

	jnz	2b

	ENDFRAME

	ret

	CFI_ENDPROC

	ENDPROC(__write_lock_failed)



ENTRY(__read_lock_failed)

	CFI_STARTPROC

	FRAME

2: 	LOCK_PREFIX

	incl	(%eax)

1:	rep; nop

	cmpl	$1,(%eax)

	js	1b

	LOCK_PREFIX

	decl	(%eax)

	js	2b

	ENDFRAME

	ret

	CFI_ENDPROC

	ENDPROC(__read_lock_failed)



#endif



#ifdef CONFIG_RWSEM_XCHGADD_ALGORITHM



/* Fix up special calling conventions */

ENTRY(call_rwsem_down_read_failed)

	CFI_STARTPROC
@@ -120,5 +78,3 @@ ENTRY(call_rwsem_downgrade_wake) 
	ret

	CFI_ENDPROC

	ENDPROC(call_rwsem_downgrade_wake)
 


#endif