Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f8e92fb4 authored by Ingo Molnar's avatar Ingo Molnar
Browse files

Merge tag 'alternatives_padding' of... 

Merge tag 'alternatives_padding' of git://git.kernel.org/pub/scm/linux/kernel/git/bp/bp

 into x86/asm

Pull alternative instructions framework improvements from Borislav Petkov:

 "A more involved rework of the alternatives framework to be able to
  pad instructions and thus make using the alternatives macros more
  straightforward and without having to figure out old and new instruction
  sizes but have the toolchain figure that out for us.

  Furthermore, it optimizes JMPs used so that fetch and decode can be
  relieved with smaller versions of the JMPs, where possible.

  Some stats:

    x86_64 defconfig:

    Alternatives sites total:               2478
    Total padding added (in Bytes):         6051

  The padding is currently done for:

    X86_FEATURE_ALWAYS
    X86_FEATURE_ERMS
    X86_FEATURE_LFENCE_RDTSC
    X86_FEATURE_MFENCE_RDTSC
    X86_FEATURE_SMAP

  This is with the latest version of the patchset. Of course, on each
  machine the alternatives sites actually being patched are a proper
  subset of the total number."

Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parents d2c032e3 dfecb95c

arch/x86/include/asm/alternative-asm.h

+42 −1
Original line number Diff line number Diff line
@@ -18,12 +18,53 @@ 
	.endm

#endif



.macro altinstruction_entry orig alt feature orig_len alt_len

.macro altinstruction_entry orig alt feature orig_len alt_len pad_len

	.long \orig - .

	.long \alt - .

	.word \feature

	.byte \orig_len

	.byte \alt_len

	.byte \pad_len

.endm



.macro ALTERNATIVE oldinstr, newinstr, feature

140:

	\oldinstr

141:

	.skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90

142:



	.pushsection .altinstructions,"a"

	altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f,142b-141b

	.popsection



	.pushsection .altinstr_replacement,"ax"

143:

	\newinstr

144:

	.popsection

.endm



.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2

140:

	\oldinstr

141:

	.skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90

	.skip -(((145f-144f)-(144f-143f)-(141b-140b)) > 0) * ((145f-144f)-(144f-143f)-(141b-140b)),0x90

142:



	.pushsection .altinstructions,"a"

	altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f,142b-141b

	altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f,142b-141b

	.popsection



	.pushsection .altinstr_replacement,"ax"

143:

	\newinstr1

144:

	\newinstr2

145:

	.popsection

.endm



#endif  /*  __ASSEMBLY__  */

arch/x86/include/asm/alternative.h

+40 −25
Original line number Diff line number Diff line
@@ -48,8 +48,9 @@ struct alt_instr { 
	s32 repl_offset;	/* offset to replacement instruction */

	u16 cpuid;		/* cpuid bit set for replacement */

	u8  instrlen;		/* length of original instruction */

	u8  replacementlen;	/* length of new instruction, <= instrlen */

};

	u8  replacementlen;	/* length of new instruction */

	u8  padlen;		/* length of build-time padding */

} __packed;



extern void alternative_instructions(void);

extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
@@ -76,50 +77,61 @@ static inline int alternatives_text_reserved(void *start, void *end) 
}

#endif	/* CONFIG_SMP */



#define OLDINSTR(oldinstr)	"661:\n\t" oldinstr "\n662:\n"



#define b_replacement(number)	"663"#number

#define e_replacement(number)	"664"#number

#define b_replacement(num)	"664"#num

#define e_replacement(num)	"665"#num



#define alt_end_marker		"663"

#define alt_slen		"662b-661b"

#define alt_rlen(number) e_replacement(number)"f-"b_replacement(number)"f"

#define alt_pad_len		alt_end_marker"b-662b"

#define alt_total_slen		alt_end_marker"b-661b"

#define alt_rlen(num)		e_replacement(num)"f-"b_replacement(num)"f"



#define __OLDINSTR(oldinstr, num)					\

	"661:\n\t" oldinstr "\n662:\n"					\

	".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * "		\

		"((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"



#define OLDINSTR(oldinstr, num)						\

	__OLDINSTR(oldinstr, num)					\

	alt_end_marker ":\n"



/*

 * Pad the second replacement alternative with additional NOPs if it is

 * additionally longer than the first replacement alternative.

 */

#define OLDINSTR_2(oldinstr, num1, num2)					\

	__OLDINSTR(oldinstr, num1)						\

	".skip -(((" alt_rlen(num2) ")-(" alt_rlen(num1) ")-(662b-661b)) > 0) * " \

		"((" alt_rlen(num2) ")-(" alt_rlen(num1) ")-(662b-661b)),0x90\n"  \

	alt_end_marker ":\n"



#define ALTINSTR_ENTRY(feature, number)					      \

#define ALTINSTR_ENTRY(feature, num)					      \

	" .long 661b - .\n"				/* label           */ \

	" .long " b_replacement(number)"f - .\n"	/* new instruction */ \

	" .long " b_replacement(num)"f - .\n"		/* new instruction */ \

	" .word " __stringify(feature) "\n"		/* feature bit     */ \

	" .byte " alt_slen "\n"				/* source len      */ \

	" .byte " alt_rlen(number) "\n"			/* replacement len */



#define DISCARD_ENTRY(number)				/* rlen <= slen */    \

	" .byte 0xff + (" alt_rlen(number) ") - (" alt_slen ")\n"

	" .byte " alt_total_slen "\n"			/* source len      */ \

	" .byte " alt_rlen(num) "\n"			/* replacement len */ \

	" .byte " alt_pad_len "\n"			/* pad len */



#define ALTINSTR_REPLACEMENT(newinstr, feature, number)	/* replacement */     \

	b_replacement(number)":\n\t" newinstr "\n" e_replacement(number) ":\n\t"

#define ALTINSTR_REPLACEMENT(newinstr, feature, num)	/* replacement */     \

	b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"



/* alternative assembly primitive: */

#define ALTERNATIVE(oldinstr, newinstr, feature)			\

	OLDINSTR(oldinstr)						\

	OLDINSTR(oldinstr, 1)						\

	".pushsection .altinstructions,\"a\"\n"				\

	ALTINSTR_ENTRY(feature, 1)					\

	".popsection\n"							\

	".pushsection .discard,\"aw\",@progbits\n"			\

	DISCARD_ENTRY(1)						\

	".popsection\n"							\

	".pushsection .altinstr_replacement, \"ax\"\n"			\

	ALTINSTR_REPLACEMENT(newinstr, feature, 1)			\

	".popsection"



#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\

	OLDINSTR(oldinstr)						\

	OLDINSTR_2(oldinstr, 1, 2)					\

	".pushsection .altinstructions,\"a\"\n"				\

	ALTINSTR_ENTRY(feature1, 1)					\

	ALTINSTR_ENTRY(feature2, 2)					\

	".popsection\n"							\

	".pushsection .discard,\"aw\",@progbits\n"			\

	DISCARD_ENTRY(1)						\

	DISCARD_ENTRY(2)						\

	".popsection\n"							\

	".pushsection .altinstr_replacement, \"ax\"\n"			\

	ALTINSTR_REPLACEMENT(newinstr1, feature1, 1)			\

	ALTINSTR_REPLACEMENT(newinstr2, feature2, 2)			\
@@ -146,6 +158,9 @@ static inline int alternatives_text_reserved(void *start, void *end) 
#define alternative(oldinstr, newinstr, feature)			\

	asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")



#define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \

	asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")



/*

 * Alternative inline assembly with input.

 *

arch/x86/include/asm/apic.h

+1 −1
Original line number Diff line number Diff line
@@ -91,7 +91,7 @@ static inline void native_apic_mem_write(u32 reg, u32 v) 
{

	volatile u32 *addr = (volatile u32 *)(APIC_BASE + reg);



	alternative_io("movl %0, %1", "xchgl %0, %1", X86_BUG_11AP,

	alternative_io("movl %0, %P1", "xchgl %0, %P1", X86_BUG_11AP,

		       ASM_OUTPUT2("=r" (v), "=m" (*addr)),

		       ASM_OUTPUT2("0" (v), "m" (*addr)));

}

arch/x86/include/asm/barrier.h

+2 −4
Original line number Diff line number Diff line
@@ -95,13 +95,11 @@ do { \ 
 * Stop RDTSC speculation. This is needed when you need to use RDTSC

 * (or get_cycles or vread that possibly accesses the TSC) in a defined

 * code region.

 *

 * (Could use an alternative three way for this if there was one.)

 */

static __always_inline void rdtsc_barrier(void)

{

	alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);

	alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);

	alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC,

			  "lfence", X86_FEATURE_LFENCE_RDTSC);

}



#endif /* _ASM_X86_BARRIER_H */

arch/x86/include/asm/cpufeature.h

+17 −13
Original line number Diff line number Diff line
@@ -419,6 +419,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit) 
			 " .word %P0\n"		/* 1: do replace */

			 " .byte 2b - 1b\n"	/* source len */

			 " .byte 0\n"		/* replacement len */

			 " .byte 0\n"		/* pad len */

			 ".previous\n"

			 /* skipping size check since replacement size = 0 */

			 : : "i" (X86_FEATURE_ALWAYS) : : t_warn);
@@ -433,6 +434,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit) 
			 " .word %P0\n"		/* feature bit */

			 " .byte 2b - 1b\n"	/* source len */

			 " .byte 0\n"		/* replacement len */

			 " .byte 0\n"		/* pad len */

			 ".previous\n"

			 /* skipping size check since replacement size = 0 */

			 : : "i" (bit) : : t_no);
@@ -458,6 +460,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit) 
			     " .word %P1\n"		/* feature bit */

			     " .byte 2b - 1b\n"		/* source len */

			     " .byte 4f - 3f\n"		/* replacement len */

			     " .byte 0\n"		/* pad len */

			     ".previous\n"

			     ".section .discard,\"aw\",@progbits\n"

			     " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
@@ -484,31 +487,30 @@ static __always_inline __pure bool __static_cpu_has(u16 bit) 
static __always_inline __pure bool _static_cpu_has_safe(u16 bit)

{

#ifdef CC_HAVE_ASM_GOTO

/*

 * We need to spell the jumps to the compiler because, depending on the offset,

 * the replacement jump can be bigger than the original jump, and this we cannot

 * have. Thus, we force the jump to the widest, 4-byte, signed relative

 * offset even though the last would often fit in less bytes.

 */

		asm_volatile_goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n"

		asm_volatile_goto("1: jmp %l[t_dynamic]\n"

			 "2:\n"

			 ".skip -(((5f-4f) - (2b-1b)) > 0) * "

			         "((5f-4f) - (2b-1b)),0x90\n"

			 "3:\n"

			 ".section .altinstructions,\"a\"\n"

			 " .long 1b - .\n"		/* src offset */

			 " .long 3f - .\n"		/* repl offset */

			 " .long 4f - .\n"		/* repl offset */

			 " .word %P1\n"			/* always replace */

			 " .byte 2b - 1b\n"		/* src len */

			 " .byte 4f - 3f\n"		/* repl len */

			 " .byte 3b - 1b\n"		/* src len */

			 " .byte 5f - 4f\n"		/* repl len */

			 " .byte 3b - 2b\n"		/* pad len */

			 ".previous\n"

			 ".section .altinstr_replacement,\"ax\"\n"

			 "3: .byte 0xe9\n .long %l[t_no] - 2b\n"

			 "4:\n"

			 "4: jmp %l[t_no]\n"

			 "5:\n"

			 ".previous\n"

			 ".section .altinstructions,\"a\"\n"

			 " .long 1b - .\n"		/* src offset */

			 " .long 0\n"			/* no replacement */

			 " .word %P0\n"			/* feature bit */

			 " .byte 2b - 1b\n"		/* src len */

			 " .byte 3b - 1b\n"		/* src len */

			 " .byte 0\n"			/* repl len */

			 " .byte 0\n"			/* pad len */

			 ".previous\n"

			 : : "i" (bit), "i" (X86_FEATURE_ALWAYS)

			 : : t_dynamic, t_no);
@@ -528,6 +530,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit) 
			     " .word %P2\n"		/* always replace */

			     " .byte 2b - 1b\n"		/* source len */

			     " .byte 4f - 3f\n"		/* replacement len */

			     " .byte 0\n"		/* pad len */

			     ".previous\n"

			     ".section .discard,\"aw\",@progbits\n"

			     " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
@@ -542,6 +545,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit) 
			     " .word %P1\n"		/* feature bit */

			     " .byte 4b - 3b\n"		/* src len */

			     " .byte 6f - 5f\n"		/* repl len */

			     " .byte 0\n"		/* pad len */

			     ".previous\n"

			     ".section .discard,\"aw\",@progbits\n"

			     " .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */