Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1b1d9258 authored by Jan Beulich's avatar Jan Beulich Committed by Ingo Molnar
Browse files

x86-64: Modify copy_user_generic() alternatives mechanism 



In order to avoid unnecessary chains of branches, rather than
implementing copy_user_generic() as a function consisting of
just a single (possibly patched) branch, instead properly deal
with patching call instructions in the alternative instructions
framework, and move the patching into the callers.

As a follow-on, one could also introduce something like
__EXPORT_SYMBOL_ALT() to avoid patching call sites in modules.

Signed-off-by: default avatarJan Beulich <jbeulich@novell.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
LKML-Reference: <4B2BB8180200007800026AE7@vpn.id2.novell.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 499a5f1e

arch/x86/include/asm/alternative.h

+6 −1
Original line number Diff line number Diff line
@@ -125,11 +125,16 @@ static inline void alternatives_smp_switch(int smp) {} 
	asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)		\

		: output : "i" (0), ## input)



/* Like alternative_io, but for replacing a direct call with another one. */

#define alternative_call(oldfunc, newfunc, feature, output, input...)	\

	asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \

		: output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)



/*

 * use this macro(s) if you need more than one output parameter

 * in alternative_io

 */

#define ASM_OUTPUT2(a, b) a, b

#define ASM_OUTPUT2(a...) a



struct paravirt_patch_site;

#ifdef CONFIG_PARAVIRT

arch/x86/include/asm/uaccess_64.h

+20 −1
Original line number Diff line number Diff line
@@ -8,6 +8,8 @@ 
#include <linux/errno.h>

#include <linux/prefetch.h>

#include <linux/lockdep.h>

#include <asm/alternative.h>

#include <asm/cpufeature.h>

#include <asm/page.h>



/*
@@ -16,7 +18,24 @@ 


/* Handles exceptions in both to and from, but doesn't do access_ok */

__must_check unsigned long

copy_user_generic(void *to, const void *from, unsigned len);

copy_user_generic_string(void *to, const void *from, unsigned len);

__must_check unsigned long

copy_user_generic_unrolled(void *to, const void *from, unsigned len);



static __always_inline __must_check unsigned long

copy_user_generic(void *to, const void *from, unsigned len)

{

	unsigned ret;



	alternative_call(copy_user_generic_unrolled,

			 copy_user_generic_string,

			 X86_FEATURE_REP_GOOD,

			 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),

				     "=d" (len)),

			 "1" (to), "2" (from), "3" (len)

			 : "memory", "rcx", "r8", "r9", "r10", "r11");

	return ret;

}



__must_check unsigned long

_copy_to_user(void __user *to, const void *from, unsigned len);

arch/x86/kernel/alternative.c

+3 −1
Original line number Diff line number Diff line
@@ -205,7 +205,7 @@ void __init_or_module apply_alternatives(struct alt_instr *start, 
					 struct alt_instr *end)

{

	struct alt_instr *a;

	char insnbuf[MAX_PATCH_LEN];

	u8 insnbuf[MAX_PATCH_LEN];



	DPRINTK("%s: alt table %p -> %p\n", __func__, start, end);

	for (a = start; a < end; a++) {
@@ -223,6 +223,8 @@ void __init_or_module apply_alternatives(struct alt_instr *start, 
		}

#endif

		memcpy(insnbuf, a->replacement, a->replacementlen);

		if (*insnbuf == 0xe8 && a->replacementlen == 5)

		    *(s32 *)(insnbuf + 1) += a->replacement - a->instr;

		add_nops(insnbuf + a->replacementlen,

			 a->instrlen - a->replacementlen);

		text_poke_early(instr, insnbuf, a->instrlen);

arch/x86/kernel/x8664_ksyms_64.c

+2 −1
Original line number Diff line number Diff line
@@ -26,7 +26,8 @@ EXPORT_SYMBOL(__put_user_2); 
EXPORT_SYMBOL(__put_user_4);

EXPORT_SYMBOL(__put_user_8);



EXPORT_SYMBOL(copy_user_generic);

EXPORT_SYMBOL(copy_user_generic_string);

EXPORT_SYMBOL(copy_user_generic_unrolled);

EXPORT_SYMBOL(__copy_user_nocache);

EXPORT_SYMBOL(_copy_from_user);

EXPORT_SYMBOL(_copy_to_user);

arch/x86/lib/copy_user_64.S

+0 −6
Original line number Diff line number Diff line
@@ -90,12 +90,6 @@ ENTRY(_copy_from_user) 
	CFI_ENDPROC

ENDPROC(_copy_from_user)



ENTRY(copy_user_generic)

	CFI_STARTPROC

	ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string

	CFI_ENDPROC

ENDPROC(copy_user_generic)



	.section .fixup,"ax"

	/* must zero dest */

ENTRY(bad_from_user)