Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6ea98b4b authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'x86-alternatives-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 alternative instruction updates from Ingo Molnar:
 "Small RDTSCP opimization, enabled by the newly added ALTERNATIVE_3(),
  and other small improvements"

* 'x86-alternatives-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/TSC: Use RDTSCP
  x86/alternatives: Add an ALTERNATIVE_3() macro
  x86/alternatives: Print containing function
  x86/alternatives: Add macro comments
parents 45802da0 093ae8f9

arch/x86/include/asm/alternative.h

+32 −7
Original line number Diff line number Diff line
@@ -94,13 +94,12 @@ static inline int alternatives_text_reserved(void *start, void *end) 
#define alt_total_slen		alt_end_marker"b-661b"

#define alt_rlen(num)		e_replacement(num)"f-"b_replacement(num)"f"



#define __OLDINSTR(oldinstr, num)					\

#define OLDINSTR(oldinstr, num)						\

	"# ALT: oldnstr\n"						\

	"661:\n\t" oldinstr "\n662:\n"					\

	"# ALT: padding\n"						\

	".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * "		\

		"((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"



#define OLDINSTR(oldinstr, num)						\

	__OLDINSTR(oldinstr, num)					\

		"((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"		\

	alt_end_marker ":\n"



/*
@@ -116,11 +115,23 @@ static inline int alternatives_text_reserved(void *start, void *end) 
 * additionally longer than the first replacement alternative.

 */

#define OLDINSTR_2(oldinstr, num1, num2) \

	"# ALT: oldinstr2\n"									\

	"661:\n\t" oldinstr "\n662:\n"								\

	"# ALT: padding2\n"									\

	".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * "	\

		"(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n"	\

	alt_end_marker ":\n"



#define OLDINSTR_3(oldinsn, n1, n2, n3)								\

	"# ALT: oldinstr3\n"									\

	"661:\n\t" oldinsn "\n662:\n"								\

	"# ALT: padding3\n"									\

	".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3))	\

		" - (" alt_slen ")) > 0) * "							\

		"(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3))	\

		" - (" alt_slen ")), 0x90\n"							\

	alt_end_marker ":\n"



#define ALTINSTR_ENTRY(feature, num)					      \

	" .long 661b - .\n"				/* label           */ \

	" .long " b_replacement(num)"f - .\n"		/* new instruction */ \
@@ -130,7 +141,8 @@ static inline int alternatives_text_reserved(void *start, void *end) 
	" .byte " alt_pad_len "\n"			/* pad len */



#define ALTINSTR_REPLACEMENT(newinstr, feature, num)	/* replacement */	\

	b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"

	"# ALT: replacement " #num "\n"						\

	b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n"



/* alternative assembly primitive: */

#define ALTERNATIVE(oldinstr, newinstr, feature)			\
@@ -153,6 +165,19 @@ static inline int alternatives_text_reserved(void *start, void *end) 
	ALTINSTR_REPLACEMENT(newinstr2, feature2, 2)			\

	".popsection\n"



#define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \

	OLDINSTR_3(oldinsn, 1, 2, 3)						\

	".pushsection .altinstructions,\"a\"\n"					\

	ALTINSTR_ENTRY(feat1, 1)						\

	ALTINSTR_ENTRY(feat2, 2)						\

	ALTINSTR_ENTRY(feat3, 3)						\

	".popsection\n"								\

	".pushsection .altinstr_replacement, \"ax\"\n"				\

	ALTINSTR_REPLACEMENT(newinsn1, feat1, 1)				\

	ALTINSTR_REPLACEMENT(newinsn2, feat2, 2)				\

	ALTINSTR_REPLACEMENT(newinsn3, feat3, 3)				\

	".popsection\n"



/*

 * Alternative instructions for different CPU types or capabilities.

 *

arch/x86/include/asm/msr.h

+14 −2
Original line number Diff line number Diff line
@@ -217,6 +217,8 @@ static __always_inline unsigned long long rdtsc(void) 
 */

static __always_inline unsigned long long rdtsc_ordered(void)

{

	DECLARE_ARGS(val, low, high);



	/*

	 * The RDTSC instruction is not ordered relative to memory

	 * access.  The Intel SDM and the AMD APM are both vague on this
@@ -227,9 +229,19 @@ static __always_inline unsigned long long rdtsc_ordered(void) 
	 * ordering guarantees as reading from a global memory location

	 * that some other imaginary CPU is updating continuously with a

	 * time stamp.

	 *

	 * Thus, use the preferred barrier on the respective CPU, aiming for

	 * RDTSCP as the default.

	 */

	barrier_nospec();

	return rdtsc();

	asm volatile(ALTERNATIVE_3("rdtsc",

				   "mfence; rdtsc", X86_FEATURE_MFENCE_RDTSC,

				   "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,

				   "rdtscp", X86_FEATURE_RDTSCP)

			: EAX_EDX_RET(val, low, high)

			/* RDTSCP clobbers ECX with MSR_TSC_AUX. */

			:: "ecx");



	return EAX_EDX_VAL(val, low, high);

}



static inline unsigned long long native_read_pmc(int counter)

arch/x86/kernel/alternative.c

+2 −2
Original line number Diff line number Diff line
@@ -394,10 +394,10 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, 
			continue;

		}



		DPRINTK("feat: %d*32+%d, old: (%px len: %d), repl: (%px, len: %d), pad: %d",

		DPRINTK("feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d), pad: %d",

			a->cpuid >> 5,

			a->cpuid & 0x1f,

			instr, a->instrlen,

			instr, instr, a->instrlen,

			replacement, a->replacementlen, a->padlen);



		DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);