Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ee9737c9 authored by Toshi Kani's avatar Toshi Kani Committed by Ingo Molnar
Browse files

x86/uaccess/64: Make the __copy_user_nocache() assembly code more readable 



Add comments to __copy_user_nocache() to clarify its procedures
and alignment requirements.

Also change numeric branch target labels to named local labels.

No code changed:

 arch/x86/lib/copy_user_64.o:

    text    data     bss     dec     hex filename
    1239       0       0    1239     4d7 copy_user_64.o.before
    1239       0       0    1239     4d7 copy_user_64.o.after

 md5:
    58bed94c2db98c1ca9a2d46d0680aaae  copy_user_64.o.before.asm
    58bed94c2db98c1ca9a2d46d0680aaae  copy_user_64.o.after.asm

Signed-off-by: default avatarToshi Kani <toshi.kani@hpe.com>
Cc: <stable@vger.kernel.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Luis R. Rodriguez <mcgrof@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Toshi Kani <toshi.kani@hp.com>
Cc: brian.boylston@hpe.com
Cc: dan.j.williams@intel.com
Cc: linux-nvdimm@lists.01.org
Cc: micah.parrish@hpe.com
Cc: ross.zwisler@linux.intel.com
Cc: vishal.l.verma@intel.com
Link: http://lkml.kernel.org/r/1455225857-12039-2-git-send-email-toshi.kani@hpe.com


[ Small readability edits and added object file comparison. ]
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 02a5f765

arch/x86/lib/copy_user_64.S

+73 −41
Original line number Diff line number Diff line
@@ -232,17 +232,30 @@ ENDPROC(copy_user_enhanced_fast_string) 


/*

 * copy_user_nocache - Uncached memory copy with exception handling

 * This will force destination/source out of cache for more performance.

 * This will force destination out of cache for more performance.

 *

 * Note: Cached memory copy is used when destination or size is not

 * naturally aligned. That is:

 *  - Require 8-byte alignment when size is 8 bytes or larger.

 */

ENTRY(__copy_user_nocache)

	ASM_STAC



	/* If size is less than 8 bytes, go to byte copy */

	cmpl $8,%edx

	jb 20f		/* less then 8 bytes, go to byte copy loop */

	jb .L_1b_cache_copy_entry



	/* If destination is not 8-byte aligned, "cache" copy to align it */

	ALIGN_DESTINATION



	/* Set 4x8-byte copy count and remainder */

	movl %edx,%ecx

	andl $63,%edx

	shrl $6,%ecx

	jz 17f

	jz .L_8b_nocache_copy_entry	/* jump if count is 0 */



	/* Perform 4x8-byte nocache loop-copy */

.L_4x8b_nocache_copy_loop:

1:	movq (%rsi),%r8

2:	movq 1*8(%rsi),%r9

3:	movq 2*8(%rsi),%r10
@@ -262,60 +275,79 @@ ENTRY(__copy_user_nocache) 
	leaq 64(%rsi),%rsi

	leaq 64(%rdi),%rdi

	decl %ecx

	jnz 1b

17:	movl %edx,%ecx

	jnz .L_4x8b_nocache_copy_loop



	/* Set 8-byte copy count and remainder */

.L_8b_nocache_copy_entry:

	movl %edx,%ecx

	andl $7,%edx

	shrl $3,%ecx

	jz 20f

18:	movq (%rsi),%r8

19:	movnti %r8,(%rdi)

	jz .L_1b_cache_copy_entry	/* jump if count is 0 */



	/* Perform 8-byte nocache loop-copy */

.L_8b_nocache_copy_loop:

20:	movq (%rsi),%r8

21:	movnti %r8,(%rdi)

	leaq 8(%rsi),%rsi

	leaq 8(%rdi),%rdi

	decl %ecx

	jnz 18b

20:	andl %edx,%edx

	jz 23f

	jnz .L_8b_nocache_copy_loop



	/* If no byte left, we're done */

.L_1b_cache_copy_entry:

	andl %edx,%edx

	jz .L_finish_copy



	/* Perform byte "cache" loop-copy for the remainder */

	movl %edx,%ecx

21:	movb (%rsi),%al

22:	movb %al,(%rdi)

.L_1b_cache_copy_loop:

40:	movb (%rsi),%al

41:	movb %al,(%rdi)

	incq %rsi

	incq %rdi

	decl %ecx

	jnz 21b

23:	xorl %eax,%eax

	jnz .L_1b_cache_copy_loop



	/* Finished copying; fence the prior stores */

.L_finish_copy:

	xorl %eax,%eax

	ASM_CLAC

	sfence

	ret



	.section .fixup,"ax"

30:	shll $6,%ecx

.L_fixup_4x8b_copy:

	shll $6,%ecx

	addl %ecx,%edx

	jmp 60f

40:	lea (%rdx,%rcx,8),%rdx

	jmp 60f

50:	movl %ecx,%edx

60:	sfence

	jmp .L_fixup_handle_tail

.L_fixup_8b_copy:

	lea (%rdx,%rcx,8),%rdx

	jmp .L_fixup_handle_tail

.L_fixup_1b_copy:

	movl %ecx,%edx

.L_fixup_handle_tail:

	sfence

	jmp copy_user_handle_tail

	.previous



	_ASM_EXTABLE(1b,30b)

	_ASM_EXTABLE(2b,30b)

	_ASM_EXTABLE(3b,30b)

	_ASM_EXTABLE(4b,30b)

	_ASM_EXTABLE(5b,30b)

	_ASM_EXTABLE(6b,30b)

	_ASM_EXTABLE(7b,30b)

	_ASM_EXTABLE(8b,30b)

	_ASM_EXTABLE(9b,30b)

	_ASM_EXTABLE(10b,30b)

	_ASM_EXTABLE(11b,30b)

	_ASM_EXTABLE(12b,30b)

	_ASM_EXTABLE(13b,30b)

	_ASM_EXTABLE(14b,30b)

	_ASM_EXTABLE(15b,30b)

	_ASM_EXTABLE(16b,30b)

	_ASM_EXTABLE(18b,40b)

	_ASM_EXTABLE(19b,40b)

	_ASM_EXTABLE(21b,50b)

	_ASM_EXTABLE(22b,50b)

	_ASM_EXTABLE(1b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(2b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(3b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(4b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(5b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(6b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(7b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(8b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(9b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(10b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(11b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(12b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(13b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(14b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(15b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(16b,.L_fixup_4x8b_copy)

	_ASM_EXTABLE(20b,.L_fixup_8b_copy)

	_ASM_EXTABLE(21b,.L_fixup_8b_copy)

	_ASM_EXTABLE(40b,.L_fixup_1b_copy)

	_ASM_EXTABLE(41b,.L_fixup_1b_copy)

ENDPROC(__copy_user_nocache)