Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3022d734 authored by Andi Kleen's avatar Andi Kleen Committed by Andi Kleen
Browse files

[PATCH] Fix zeroing on exception in copy_*_user 



- Don't zero for __copy_from_user_inatomic following i386.
This will prevent spurious zeros for parallel file system writers when
one does a exception
- The string instruction version didn't zero the output on
exception. Oops.

Also I cleaned up the code a bit while I was at it and added a minor
optimization to the string instruction path.

Signed-off-by: default avatarAndi Kleen <ak@suse.de>
parent f0f4c343

arch/x86_64/kernel/x8664_ksyms.c

+1 −0
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ EXPORT_SYMBOL(__put_user_8); 
EXPORT_SYMBOL(copy_user_generic);

EXPORT_SYMBOL(copy_from_user);

EXPORT_SYMBOL(copy_to_user);

EXPORT_SYMBOL(__copy_from_user_inatomic);



EXPORT_SYMBOL(copy_page);

EXPORT_SYMBOL(clear_page);

arch/x86_64/lib/copy_user.S

+79 −45
Original line number Diff line number Diff line
@@ -14,6 +14,25 @@ 
#include <asm/thread_info.h>

#include <asm/cpufeature.h>



	.macro ALTERNATIVE_JUMP feature,orig,alt

0:

	.byte 0xe9	/* 32bit jump */

	.long \orig-1f	/* by default jump to orig */

1:

	.section .altinstr_replacement,"ax"

2:	.byte 0xe9	             /* near jump with 32bit immediate */

	.long \alt-1b /* offset */   /* or alternatively to alt */

	.previous

	.section .altinstructions,"a"

	.align 8

	.quad  0b

	.quad  2b

	.byte  \feature		     /* when feature is set */

	.byte  5

	.byte  5

	.previous

	.endm



/* Standard copy_to_user with segment limit checking */		

ENTRY(copy_to_user)

	CFI_STARTPROC
@@ -23,25 +42,21 @@ ENTRY(copy_to_user) 
	jc  bad_to_user

	cmpq threadinfo_addr_limit(%rax),%rcx

	jae bad_to_user

2:

	.byte 0xe9	/* 32bit jump */

	.long .Lcug-1f

1:

	xorl %eax,%eax	/* clear zero flag */

	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string

	CFI_ENDPROC

ENDPROC(copy_to_user)



	.section .altinstr_replacement,"ax"

3:	.byte 0xe9			/* replacement jmp with 32 bit immediate */

	.long copy_user_generic_c-1b	/* offset */

	.previous

	.section .altinstructions,"a"

	.align 8

	.quad  2b

	.quad  3b

	.byte  X86_FEATURE_REP_GOOD

	.byte  5

	.byte  5

	.previous

ENTRY(copy_user_generic)

	CFI_STARTPROC

	movl $1,%ecx	/* set zero flag */

	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string

	CFI_ENDPROC



ENTRY(__copy_from_user_inatomic)

	CFI_STARTPROC

	xorl %ecx,%ecx	/* clear zero flag */

	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string

	CFI_ENDPROC



/* Standard copy_from_user with segment limit checking */	

ENTRY(copy_from_user)
@@ -52,7 +67,8 @@ ENTRY(copy_from_user) 
	jc  bad_from_user

	cmpq threadinfo_addr_limit(%rax),%rcx

	jae  bad_from_user

	/* FALL THROUGH to copy_user_generic */

	movl $1,%ecx	/* set zero flag */

	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string

	CFI_ENDPROC

ENDPROC(copy_from_user)
@@ -73,37 +89,26 @@ END(bad_from_user) 
	

		

/*

 * copy_user_generic - memory copy with exception handling.

 * copy_user_generic_unrolled - memory copy with exception handling.

 * This version is for CPUs like P4 that don't have efficient micro code for rep movsq

 * 	

 * Input:	

 * rdi destination

 * rsi source

 * rdx count

 * ecx zero flag -- if true zero destination on error

 *

 * Output:		

 * eax uncopied bytes or 0 if successful.

 */

ENTRY(copy_user_generic)

ENTRY(copy_user_generic_unrolled)

	CFI_STARTPROC

	.byte 0x66,0x66,0x90	/* 5 byte nop for replacement jump */

	.byte 0x66,0x90

1:

	.section .altinstr_replacement,"ax"

2:	.byte 0xe9	             /* near jump with 32bit immediate */

	.long copy_user_generic_c-1b /* offset */

	.previous

	.section .altinstructions,"a"

	.align 8

	.quad  copy_user_generic

	.quad  2b

	.byte  X86_FEATURE_REP_GOOD

	.byte  5

	.byte  5

	.previous

.Lcug:

	pushq %rbx

	CFI_ADJUST_CFA_OFFSET 8

	CFI_REL_OFFSET rbx, 0

	pushq %rcx

	CFI_ADJUST_CFA_OFFSET 8

	CFI_REL_OFFSET rcx, 0

	xorl %eax,%eax		/*zero for the exception handler */



#ifdef FIX_ALIGNMENT
@@ -179,6 +184,9 @@ ENTRY(copy_user_generic) 


	CFI_REMEMBER_STATE

.Lende:

	popq %rcx

	CFI_ADJUST_CFA_OFFSET -8

	CFI_RESTORE rcx

	popq %rbx

	CFI_ADJUST_CFA_OFFSET -8

	CFI_RESTORE rbx
@@ -265,6 +273,8 @@ ENTRY(copy_user_generic) 
	addl %ecx,%edx

	/* edx: bytes to zero, rdi: dest, eax:zero */

.Lzero_rest:

	cmpl $0,(%rsp)

	jz   .Le_zero

	movq %rdx,%rcx

.Le_byte:

	xorl %eax,%eax
@@ -286,6 +296,7 @@ ENDPROC(copy_user_generic) 
 /* rdi	destination

  * rsi source

  * rdx count

  * ecx zero flag

  *

  * Output:

  * eax uncopied bytes or 0 if successfull.
@@ -297,24 +308,47 @@ ENDPROC(copy_user_generic) 
  * errata with rep movsq > 4GB. If someone feels the need to fix

  * this please consider this.

  */

copy_user_generic_c:

ENTRY(copy_user_generic_string)

	CFI_STARTPROC

	movl %ecx,%r8d		/* save zero flag */

	movl %edx,%ecx

	shrl $3,%ecx

	andl $7,%edx	

	jz   10f

1:	rep 

	movsq 

	movl %edx,%ecx

2:	rep

	movsb

4:	movl %ecx,%eax

9:	movl %ecx,%eax

	ret

3:	lea (%rdx,%rcx,8),%rax



	/* multiple of 8 byte */

10:	rep

	movsq

	xor %eax,%eax

	ret



	/* exception handling */

3:      lea (%rdx,%rcx,8),%rax	/* exception on quad loop */

	jmp 6f

5:	movl %ecx,%eax		/* exception on byte loop */

	/* eax: left over bytes */

6:	testl %r8d,%r8d		/* zero flag set? */

	jz 7f

	movl %eax,%ecx		/* initialize x86 loop counter */

	push %rax

	xorl %eax,%eax

8:	rep

	stosb 			/* zero the rest */

11:	pop %rax

7:	ret

	CFI_ENDPROC

END(copy_user_generic_c)



	.section __ex_table,"a"

	.quad 1b,3b

	.quad 2b,4b

	.quad 2b,5b

	.quad 8b,11b

	.quad 10b,3b

	.previous

include/asm-x86_64/uaccess.h

+3 −3
Original line number Diff line number Diff line
@@ -238,6 +238,7 @@ do { \ 


/* Handles exceptions in both to and from, but doesn't do access_ok */

extern unsigned long copy_user_generic(void *to, const void *from, unsigned len); 

extern unsigned long copy_user_generic_dontzero(void *to, const void *from, unsigned len);



extern unsigned long copy_to_user(void __user *to, const void *from, unsigned len); 

extern unsigned long copy_from_user(void *to, const void __user *from, unsigned len);
@@ -303,7 +304,6 @@ static __always_inline int __copy_to_user(void __user *dst, const void *src, uns 
	}

}	





static __always_inline int __copy_in_user(void __user *dst, const void __user *src, unsigned size)

{ 

       int ret = 0;
@@ -352,7 +352,7 @@ long strlen_user(const char __user *str); 
unsigned long clear_user(void __user *mem, unsigned long len);

unsigned long __clear_user(void __user *mem, unsigned long len);



#define __copy_to_user_inatomic __copy_to_user

#define __copy_from_user_inatomic __copy_from_user

extern long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size);

#define __copy_to_user_inatomic copy_user_generic



#endif /* __X86_64_UACCESS_H */