Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 76b04384 authored by David Woodhouse's avatar David Woodhouse Committed by Thomas Gleixner
Browse files

x86/retpoline: Add initial retpoline support 



Enable the use of -mindirect-branch=thunk-extern in newer GCC, and provide
the corresponding thunks. Provide assembler macros for invoking the thunks
in the same way that GCC does, from native and inline assembler.

This adds X86_FEATURE_RETPOLINE and sets it by default on all CPUs. In
some circumstances, IBRS microcode features may be used instead, and the
retpoline can be disabled.

On AMD CPUs if lfence is serialising, the retpoline can be dramatically
simplified to a simple "lfence; jmp *\reg". A future patch, after it has
been verified that lfence really is serialising in all circumstances, can
enable this by setting the X86_FEATURE_RETPOLINE_AMD feature bit in addition
to X86_FEATURE_RETPOLINE.

Do not align the retpoline in the altinstr section, because there is no
guarantee that it stays aligned when it's copied over the oldinstr during
alternative patching.

[ Andi Kleen: Rename the macros, add CONFIG_RETPOLINE option, export thunks]
[ tglx: Put actual function CALL/JMP in front of the macros, convert to
  	symbolic labels ]
[ dwmw2: Convert back to numeric labels, merge objtool fixes ]

Signed-off-by: default avatarDavid Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Acked-by: default avatarArjan van de Ven <arjan@linux.intel.com>
Acked-by: default avatarIngo Molnar <mingo@kernel.org>
Cc: gnomes@lxorguk.ukuu.org.uk
Cc: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: thomas.lendacky@amd.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Jiri Kosina <jikos@kernel.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Kees Cook <keescook@google.com>
Cc: Tim Chen <tim.c.chen@linux.intel.com>
Cc: Greg Kroah-Hartman <gregkh@linux-foundation.org>
Cc: Paul Turner <pjt@google.com>
Link: https://lkml.kernel.org/r/1515707194-20531-4-git-send-email-dwmw@amazon.co.uk
parent 258c7605

arch/x86/Kconfig

+13 −0
Original line number Diff line number Diff line
@@ -429,6 +429,19 @@ config GOLDFISH 
       def_bool y

       depends on X86_GOLDFISH



config RETPOLINE

	bool "Avoid speculative indirect branches in kernel"

	default y

	help

	  Compile kernel with the retpoline compiler options to guard against

	  kernel-to-user data leaks by avoiding speculative indirect

	  branches. Requires a compiler with -mindirect-branch=thunk-extern

	  support for full protection. The kernel may run slower.



	  Without compiler support, at least indirect branches in assembler

	  code are eliminated. Since this includes the syscall entry path,

	  it is not entirely pointless.



config INTEL_RDT

	bool "Intel Resource Director Technology support"

	default n

arch/x86/Makefile

+10 −0
Original line number Diff line number Diff line
@@ -235,6 +235,16 @@ KBUILD_CFLAGS += -Wno-sign-compare 
#

KBUILD_CFLAGS += -fno-asynchronous-unwind-tables



# Avoid indirect branches in kernel to deal with Spectre

ifdef CONFIG_RETPOLINE

    RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register)

    ifneq ($(RETPOLINE_CFLAGS),)

        KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE

    else

        $(warning CONFIG_RETPOLINE=y, but not supported by the compiler. Toolchain update recommended.)

    endif

endif



archscripts: scripts_basic

	$(Q)$(MAKE) $(build)=arch/x86/tools relocs

arch/x86/include/asm/asm-prototypes.h

+25 −0
Original line number Diff line number Diff line
@@ -11,7 +11,32 @@ 
#include <asm/pgtable.h>

#include <asm/special_insns.h>

#include <asm/preempt.h>

#include <asm/asm.h>



#ifndef CONFIG_X86_CMPXCHG64

extern void cmpxchg8b_emu(void);

#endif



#ifdef CONFIG_RETPOLINE

#ifdef CONFIG_X86_32

#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_e ## reg(void);

#else

#define INDIRECT_THUNK(reg) extern asmlinkage void __x86_indirect_thunk_r ## reg(void);

INDIRECT_THUNK(8)

INDIRECT_THUNK(9)

INDIRECT_THUNK(10)

INDIRECT_THUNK(11)

INDIRECT_THUNK(12)

INDIRECT_THUNK(13)

INDIRECT_THUNK(14)

INDIRECT_THUNK(15)

#endif

INDIRECT_THUNK(ax)

INDIRECT_THUNK(bx)

INDIRECT_THUNK(cx)

INDIRECT_THUNK(dx)

INDIRECT_THUNK(si)

INDIRECT_THUNK(di)

INDIRECT_THUNK(bp)

INDIRECT_THUNK(sp)

#endif /* CONFIG_RETPOLINE */

arch/x86/include/asm/cpufeatures.h

+2 −0
Original line number Diff line number Diff line
@@ -203,6 +203,8 @@ 
#define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */

#define X86_FEATURE_SME			( 7*32+10) /* AMD Secure Memory Encryption */

#define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */

#define X86_FEATURE_RETPOLINE		( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */

#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */

#define X86_FEATURE_INTEL_PPIN		( 7*32+14) /* Intel Processor Inventory Number */

#define X86_FEATURE_INTEL_PT		( 7*32+15) /* Intel Processor Trace */

#define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */

arch/x86/include/asm/nospec-branch.h

0 → 100644
+128 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */



#ifndef __NOSPEC_BRANCH_H__

#define __NOSPEC_BRANCH_H__



#include <asm/alternative.h>

#include <asm/alternative-asm.h>

#include <asm/cpufeatures.h>



#ifdef __ASSEMBLY__



/*

 * This should be used immediately before a retpoline alternative.  It tells

 * objtool where the retpolines are so that it can make sense of the control

 * flow by just reading the original instruction(s) and ignoring the

 * alternatives.

 */

.macro ANNOTATE_NOSPEC_ALTERNATIVE

	.Lannotate_\@:

	.pushsection .discard.nospec

	.long .Lannotate_\@ - .

	.popsection

.endm



/*

 * These are the bare retpoline primitives for indirect jmp and call.

 * Do not use these directly; they only exist to make the ALTERNATIVE

 * invocation below less ugly.

 */

.macro RETPOLINE_JMP reg:req

	call	.Ldo_rop_\@

.Lspec_trap_\@:

	pause

	jmp	.Lspec_trap_\@

.Ldo_rop_\@:

	mov	\reg, (%_ASM_SP)

	ret

.endm



/*

 * This is a wrapper around RETPOLINE_JMP so the called function in reg

 * returns to the instruction after the macro.

 */

.macro RETPOLINE_CALL reg:req

	jmp	.Ldo_call_\@

.Ldo_retpoline_jmp_\@:

	RETPOLINE_JMP \reg

.Ldo_call_\@:

	call	.Ldo_retpoline_jmp_\@

.endm



/*

 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple

 * indirect jmp/call which may be susceptible to the Spectre variant 2

 * attack.

 */

.macro JMP_NOSPEC reg:req

#ifdef CONFIG_RETPOLINE

	ANNOTATE_NOSPEC_ALTERNATIVE

	ALTERNATIVE_2 __stringify(jmp *\reg),				\

		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\

		__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD

#else

	jmp	*\reg

#endif

.endm



.macro CALL_NOSPEC reg:req

#ifdef CONFIG_RETPOLINE

	ANNOTATE_NOSPEC_ALTERNATIVE

	ALTERNATIVE_2 __stringify(call *\reg),				\

		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\

		__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD

#else

	call	*\reg

#endif

.endm



#else /* __ASSEMBLY__ */



#define ANNOTATE_NOSPEC_ALTERNATIVE				\

	"999:\n\t"						\

	".pushsection .discard.nospec\n\t"			\

	".long 999b - .\n\t"					\

	".popsection\n\t"



#if defined(CONFIG_X86_64) && defined(RETPOLINE)



/*

 * Since the inline asm uses the %V modifier which is only in newer GCC,

 * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.

 */

# define CALL_NOSPEC						\

	ANNOTATE_NOSPEC_ALTERNATIVE				\

	ALTERNATIVE(						\

	"call *%[thunk_target]\n",				\

	"call __x86_indirect_thunk_%V[thunk_target]\n",		\

	X86_FEATURE_RETPOLINE)

# define THUNK_TARGET(addr) [thunk_target] "r" (addr)



#elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)

/*

 * For i386 we use the original ret-equivalent retpoline, because

 * otherwise we'll run out of registers. We don't care about CET

 * here, anyway.

 */

# define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n",	\

	"       jmp    904f;\n"					\

	"       .align 16\n"					\

	"901:	call   903f;\n"					\

	"902:	pause;\n"					\

	"       jmp    902b;\n"					\

	"       .align 16\n"					\

	"903:	addl   $4, %%esp;\n"				\

	"       pushl  %[thunk_target];\n"			\

	"       ret;\n"						\

	"       .align 16\n"					\

	"904:	call   901b;\n",				\

	X86_FEATURE_RETPOLINE)



# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)

#else /* No retpoline */

# define CALL_NOSPEC "call *%[thunk_target]\n"

# define THUNK_TARGET(addr) [thunk_target] "rm" (addr)

#endif



#endif /* __ASSEMBLY__ */

#endif /* __NOSPEC_BRANCH_H__ */