Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d09a8e6f authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'x86-dax-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 dax updates from Ingo Molnar:
 "This contains x86 memcpy_mcsafe() fault handling improvements the
  nvdimm tree would like to make more use of"

* 'x86-dax-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/asm/memcpy_mcsafe: Define copy_to_iter_mcsafe()
  x86/asm/memcpy_mcsafe: Add write-protection-fault handling
  x86/asm/memcpy_mcsafe: Return bytes remaining
  x86/asm/memcpy_mcsafe: Add labels for __memcpy_mcsafe() write fault handling
  x86/asm/memcpy_mcsafe: Remove loop unrolling
parents 83163856 8780356e

arch/x86/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -62,6 +62,7 @@ config X86 
	select ARCH_HAS_PMEM_API		if X86_64

	select ARCH_HAS_REFCOUNT

	select ARCH_HAS_UACCESS_FLUSHCACHE	if X86_64

	select ARCH_HAS_UACCESS_MCSAFE		if X86_64

	select ARCH_HAS_SET_MEMORY

	select ARCH_HAS_SG_CHAIN

	select ARCH_HAS_STRICT_KERNEL_RWX

arch/x86/include/asm/string_64.h

+6 −4
Original line number Diff line number Diff line
@@ -116,7 +116,8 @@ int strcmp(const char *cs, const char *ct); 
#endif



#define __HAVE_ARCH_MEMCPY_MCSAFE 1

__must_check int memcpy_mcsafe_unrolled(void *dst, const void *src, size_t cnt);

__must_check unsigned long __memcpy_mcsafe(void *dst, const void *src,

		size_t cnt);

DECLARE_STATIC_KEY_FALSE(mcsafe_key);



/**
@@ -131,14 +132,15 @@ DECLARE_STATIC_KEY_FALSE(mcsafe_key); 
 * actually do machine check recovery. Everyone else can just

 * use memcpy().

 *

 * Return 0 for success, -EFAULT for fail

 * Return 0 for success, or number of bytes not copied if there was an

 * exception.

 */

static __always_inline __must_check int

static __always_inline __must_check unsigned long

memcpy_mcsafe(void *dst, const void *src, size_t cnt)

{

#ifdef CONFIG_X86_MCE

	if (static_branch_unlikely(&mcsafe_key))

		return memcpy_mcsafe_unrolled(dst, src, cnt);

		return __memcpy_mcsafe(dst, src, cnt);

	else

#endif

		memcpy(dst, src, cnt);

arch/x86/include/asm/uaccess_64.h

+14 −0
Original line number Diff line number Diff line
@@ -46,6 +46,17 @@ copy_user_generic(void *to, const void *from, unsigned len) 
	return ret;

}



static __always_inline __must_check unsigned long

copy_to_user_mcsafe(void *to, const void *from, unsigned len)

{

	unsigned long ret;



	__uaccess_begin();

	ret = memcpy_mcsafe(to, from, len);

	__uaccess_end();

	return ret;

}



static __always_inline __must_check unsigned long

raw_copy_from_user(void *dst, const void __user *src, unsigned long size)

{
@@ -194,4 +205,7 @@ __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) 
unsigned long

copy_user_handle_tail(char *to, char *from, unsigned len);



unsigned long

mcsafe_handle_tail(char *to, char *from, unsigned len);



#endif /* _ASM_X86_UACCESS_64_H */

arch/x86/lib/memcpy_64.S

+44 −58
Original line number Diff line number Diff line
@@ -184,11 +184,11 @@ ENDPROC(memcpy_orig) 


#ifndef CONFIG_UML

/*

 * memcpy_mcsafe_unrolled - memory copy with machine check exception handling

 * __memcpy_mcsafe - memory copy with machine check exception handling

 * Note that we only catch machine checks when reading the source addresses.

 * Writes to target are posted and don't generate machine checks.

 */

ENTRY(memcpy_mcsafe_unrolled)

ENTRY(__memcpy_mcsafe)

	cmpl $8, %edx

	/* Less than 8 bytes? Go to byte copy loop */

	jb .L_no_whole_words
@@ -204,58 +204,29 @@ ENTRY(memcpy_mcsafe_unrolled) 
	subl $8, %ecx

	negl %ecx

	subl %ecx, %edx

.L_copy_leading_bytes:

.L_read_leading_bytes:

	movb (%rsi), %al

.L_write_leading_bytes:

	movb %al, (%rdi)

	incq %rsi

	incq %rdi

	decl %ecx

	jnz .L_copy_leading_bytes

	jnz .L_read_leading_bytes



.L_8byte_aligned:

	/* Figure out how many whole cache lines (64-bytes) to copy */

	movl %edx, %ecx

	andl $63, %edx

	shrl $6, %ecx

	jz .L_no_whole_cache_lines



	/* Loop copying whole cache lines */

.L_cache_w0: movq (%rsi), %r8

.L_cache_w1: movq 1*8(%rsi), %r9

.L_cache_w2: movq 2*8(%rsi), %r10

.L_cache_w3: movq 3*8(%rsi), %r11

	movq %r8, (%rdi)

	movq %r9, 1*8(%rdi)

	movq %r10, 2*8(%rdi)

	movq %r11, 3*8(%rdi)

.L_cache_w4: movq 4*8(%rsi), %r8

.L_cache_w5: movq 5*8(%rsi), %r9

.L_cache_w6: movq 6*8(%rsi), %r10

.L_cache_w7: movq 7*8(%rsi), %r11

	movq %r8, 4*8(%rdi)

	movq %r9, 5*8(%rdi)

	movq %r10, 6*8(%rdi)

	movq %r11, 7*8(%rdi)

	leaq 64(%rsi), %rsi

	leaq 64(%rdi), %rdi

	decl %ecx

	jnz .L_cache_w0



	/* Are there any trailing 8-byte words? */

.L_no_whole_cache_lines:

	movl %edx, %ecx

	andl $7, %edx

	shrl $3, %ecx

	jz .L_no_whole_words



	/* Copy trailing words */

.L_copy_trailing_words:

.L_read_words:

	movq (%rsi), %r8

	mov %r8, (%rdi)

	leaq 8(%rsi), %rsi

	leaq 8(%rdi), %rdi

.L_write_words:

	movq %r8, (%rdi)

	addq $8, %rsi

	addq $8, %rdi

	decl %ecx

	jnz .L_copy_trailing_words

	jnz .L_read_words



	/* Any trailing bytes? */

.L_no_whole_words:
@@ -264,38 +235,53 @@ ENTRY(memcpy_mcsafe_unrolled) 


	/* Copy trailing bytes */

	movl %edx, %ecx

.L_copy_trailing_bytes:

.L_read_trailing_bytes:

	movb (%rsi), %al

.L_write_trailing_bytes:

	movb %al, (%rdi)

	incq %rsi

	incq %rdi

	decl %ecx

	jnz .L_copy_trailing_bytes

	jnz .L_read_trailing_bytes



	/* Copy successful. Return zero */

.L_done_memcpy_trap:

	xorq %rax, %rax

	ret

ENDPROC(memcpy_mcsafe_unrolled)

EXPORT_SYMBOL_GPL(memcpy_mcsafe_unrolled)

ENDPROC(__memcpy_mcsafe)

EXPORT_SYMBOL_GPL(__memcpy_mcsafe)



	.section .fixup, "ax"

	/* Return -EFAULT for any failure */

.L_memcpy_mcsafe_fail:

	mov	$-EFAULT, %rax

	/*

	 * Return number of bytes not copied for any failure. Note that

	 * there is no "tail" handling since the source buffer is 8-byte

	 * aligned and poison is cacheline aligned.

	 */

.E_read_words:

	shll	$3, %ecx

.E_leading_bytes:

	addl	%edx, %ecx

.E_trailing_bytes:

	mov	%ecx, %eax

	ret



	/*

	 * For write fault handling, given the destination is unaligned,

	 * we handle faults on multi-byte writes with a byte-by-byte

	 * copy up to the write-protected page.

	 */

.E_write_words:

	shll	$3, %ecx

	addl	%edx, %ecx

	movl	%ecx, %edx

	jmp mcsafe_handle_tail



	.previous



	_ASM_EXTABLE_FAULT(.L_copy_leading_bytes, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_cache_w0, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_cache_w1, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_cache_w2, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_cache_w3, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_cache_w4, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_cache_w5, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_cache_w6, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_cache_w7, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_copy_trailing_words, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_copy_trailing_bytes, .L_memcpy_mcsafe_fail)

	_ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes)

	_ASM_EXTABLE_FAULT(.L_read_words, .E_read_words)

	_ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes)

	_ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)

	_ASM_EXTABLE(.L_write_words, .E_write_words)

	_ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)

#endif

arch/x86/lib/usercopy_64.c

+21 −0
Original line number Diff line number Diff line
@@ -74,6 +74,27 @@ copy_user_handle_tail(char *to, char *from, unsigned len) 
	return len;

}



/*

 * Similar to copy_user_handle_tail, probe for the write fault point,

 * but reuse __memcpy_mcsafe in case a new read error is encountered.

 * clac() is handled in _copy_to_iter_mcsafe().

 */

__visible unsigned long

mcsafe_handle_tail(char *to, char *from, unsigned len)

{

	for (; len; --len, to++, from++) {

		/*

		 * Call the assembly routine back directly since

		 * memcpy_mcsafe() may silently fallback to memcpy.

		 */

		unsigned long rem = __memcpy_mcsafe(to, from, 1);



		if (rem)

			break;

	}

	return len;

}



#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE

/**

 * clean_cache_range - write back a cache range with CLWB