Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d0ba5027 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Greg Kroah-Hartman
Browse files

x86/speculation: Add eIBRS + Retpoline options 



commit 1e19da8522c81bf46b335f84137165741e0d82b7 upstream.

Thanks to the chaps at VUsec it is now clear that eIBRS is not
sufficient, therefore allow enabling of retpolines along with eIBRS.

Add spectre_v2=eibrs, spectre_v2=eibrs,lfence and
spectre_v2=eibrs,retpoline options to explicitly pick your preferred
means of mitigation.

Since there's new mitigations there's also user visible changes in
/sys/devices/system/cpu/vulnerabilities/spectre_v2 to reflect these
new mitigations.

  [ bp: Massage commit message, trim error messages,
    do more precise eIBRS mode checking. ]

Co-developed-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: default avatarJosh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Reviewed-by: default avatarPatrick Colp <patrick.colp@oracle.com>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
[fllinden@amazon.com: backported to 4.19 (no Hygon)]
Signed-off-by: default avatarFrank van der Linden <fllinden@amazon.com>
Signed-off-by: default avatarBen Hutchings <ben@decadent.org.uk>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent a771511c

arch/x86/include/asm/nospec-branch.h

+3 −1
Original line number Diff line number Diff line
@@ -225,7 +225,9 @@ enum spectre_v2_mitigation { 
	SPECTRE_V2_NONE,

	SPECTRE_V2_RETPOLINE,

	SPECTRE_V2_LFENCE,

	SPECTRE_V2_IBRS_ENHANCED,

	SPECTRE_V2_EIBRS,

	SPECTRE_V2_EIBRS_RETPOLINE,

	SPECTRE_V2_EIBRS_LFENCE,

};



/* The indirect branch speculation control variants */

arch/x86/kernel/cpu/bugs.c

+95 −36
Original line number Diff line number Diff line
@@ -621,6 +621,9 @@ enum spectre_v2_mitigation_cmd { 
	SPECTRE_V2_CMD_RETPOLINE,

	SPECTRE_V2_CMD_RETPOLINE_GENERIC,

	SPECTRE_V2_CMD_RETPOLINE_LFENCE,

	SPECTRE_V2_CMD_EIBRS,

	SPECTRE_V2_CMD_EIBRS_RETPOLINE,

	SPECTRE_V2_CMD_EIBRS_LFENCE,

};



enum spectre_v2_user_cmd {
@@ -693,6 +696,13 @@ spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd) 
	return SPECTRE_V2_USER_CMD_AUTO;

}



static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)

{

	return (mode == SPECTRE_V2_EIBRS ||

		mode == SPECTRE_V2_EIBRS_RETPOLINE ||

		mode == SPECTRE_V2_EIBRS_LFENCE);

}



static void __init

spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)

{
@@ -760,7 +770,7 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) 
	 */

	if (!boot_cpu_has(X86_FEATURE_STIBP) ||

	    !smt_possible ||

	    spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)

	    spectre_v2_in_eibrs_mode(spectre_v2_enabled))

		return;



	/*
@@ -782,7 +792,9 @@ static const char * const spectre_v2_strings[] = { 
	[SPECTRE_V2_NONE]			= "Vulnerable",

	[SPECTRE_V2_RETPOLINE]			= "Mitigation: Retpolines",

	[SPECTRE_V2_LFENCE]			= "Mitigation: LFENCE",

	[SPECTRE_V2_IBRS_ENHANCED]		= "Mitigation: Enhanced IBRS",

	[SPECTRE_V2_EIBRS]			= "Mitigation: Enhanced IBRS",

	[SPECTRE_V2_EIBRS_LFENCE]		= "Mitigation: Enhanced IBRS + LFENCE",

	[SPECTRE_V2_EIBRS_RETPOLINE]		= "Mitigation: Enhanced IBRS + Retpolines",

};



static const struct {
@@ -796,6 +808,9 @@ static const struct { 
	{ "retpoline,amd",	SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },

	{ "retpoline,lfence",	SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },

	{ "retpoline,generic",	SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },

	{ "eibrs",		SPECTRE_V2_CMD_EIBRS,		  false },

	{ "eibrs,lfence",	SPECTRE_V2_CMD_EIBRS_LFENCE,	  false },

	{ "eibrs,retpoline",	SPECTRE_V2_CMD_EIBRS_RETPOLINE,	  false },

	{ "auto",		SPECTRE_V2_CMD_AUTO,		  false },

};
@@ -833,15 +848,29 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 


	if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||

	     cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||

	     cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&

	     cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||

	     cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||

	     cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&

	    !IS_ENABLED(CONFIG_RETPOLINE)) {

		pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);

		pr_err("%s selected but not compiled in. Switching to AUTO select\n",

		       mitigation_options[i].option);

		return SPECTRE_V2_CMD_AUTO;

	}



	if ((cmd == SPECTRE_V2_CMD_EIBRS ||

	     cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||

	     cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&

	    !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {

		pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",

		       mitigation_options[i].option);

		return SPECTRE_V2_CMD_AUTO;

	}



	if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE) &&

	if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||

	     cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&

	    !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {

		pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", mitigation_options[i].option);

		pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",

		       mitigation_options[i].option);

		return SPECTRE_V2_CMD_AUTO;

	}
@@ -850,6 +879,24 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 
	return cmd;

}



static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)

{

	if (!IS_ENABLED(CONFIG_RETPOLINE)) {

		pr_err("Kernel not compiled with retpoline; no mitigation available!");

		return SPECTRE_V2_NONE;

	}



	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {

		if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {

			pr_err("LFENCE not serializing, switching to generic retpoline\n");

			return SPECTRE_V2_RETPOLINE;

		}

		return SPECTRE_V2_LFENCE;

	}



	return SPECTRE_V2_RETPOLINE;

}



static void __init spectre_v2_select_mitigation(void)

{

	enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
@@ -870,48 +917,60 @@ static void __init spectre_v2_select_mitigation(void) 
	case SPECTRE_V2_CMD_FORCE:

	case SPECTRE_V2_CMD_AUTO:

		if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {

			mode = SPECTRE_V2_IBRS_ENHANCED;

			/* Force it so VMEXIT will restore correctly */

			x86_spec_ctrl_base |= SPEC_CTRL_IBRS;

			wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);

			goto specv2_set_mode;

			mode = SPECTRE_V2_EIBRS;

			break;

		}

		if (IS_ENABLED(CONFIG_RETPOLINE))

			goto retpoline_auto;



		mode = spectre_v2_select_retpoline();

		break;



	case SPECTRE_V2_CMD_RETPOLINE_LFENCE:

		if (IS_ENABLED(CONFIG_RETPOLINE))

			goto retpoline_lfence;

		mode = SPECTRE_V2_LFENCE;

		break;



	case SPECTRE_V2_CMD_RETPOLINE_GENERIC:

		if (IS_ENABLED(CONFIG_RETPOLINE))

			goto retpoline_generic;

		mode = SPECTRE_V2_RETPOLINE;

		break;



	case SPECTRE_V2_CMD_RETPOLINE:

		if (IS_ENABLED(CONFIG_RETPOLINE))

			goto retpoline_auto;

		mode = spectre_v2_select_retpoline();

		break;



	case SPECTRE_V2_CMD_EIBRS:

		mode = SPECTRE_V2_EIBRS;

		break;



	case SPECTRE_V2_CMD_EIBRS_LFENCE:

		mode = SPECTRE_V2_EIBRS_LFENCE;

		break;



	case SPECTRE_V2_CMD_EIBRS_RETPOLINE:

		mode = SPECTRE_V2_EIBRS_RETPOLINE;

		break;

	}

	pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");

	return;



retpoline_auto:

	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {

	retpoline_lfence:

		if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {

			pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");

			goto retpoline_generic;

	if (spectre_v2_in_eibrs_mode(mode)) {

		/* Force it so VMEXIT will restore correctly */

		x86_spec_ctrl_base |= SPEC_CTRL_IBRS;

		wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);

	}

		mode = SPECTRE_V2_LFENCE;



	switch (mode) {

	case SPECTRE_V2_NONE:

	case SPECTRE_V2_EIBRS:

		break;



	case SPECTRE_V2_LFENCE:

	case SPECTRE_V2_EIBRS_LFENCE:

		setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);

		/* fallthrough */



	case SPECTRE_V2_RETPOLINE:

	case SPECTRE_V2_EIBRS_RETPOLINE:

		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);

	} else {

	retpoline_generic:

		mode = SPECTRE_V2_RETPOLINE;

		setup_force_cpu_cap(X86_FEATURE_RETPOLINE);

		break;

	}



specv2_set_mode:

	spectre_v2_enabled = mode;

	pr_info("%s\n", spectre_v2_strings[mode]);
@@ -937,7 +996,7 @@ static void __init spectre_v2_select_mitigation(void) 
	 * the CPU supports Enhanced IBRS, kernel might un-intentionally not

	 * enable IBRS around firmware calls.

	 */

	if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {

	if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) {

		setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);

		pr_info("Enabling Restricted Speculation for firmware calls\n");

	}
@@ -1597,7 +1656,7 @@ static ssize_t tsx_async_abort_show_state(char *buf) 


static char *stibp_state(void)

{

	if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)

	if (spectre_v2_in_eibrs_mode(spectre_v2_enabled))

		return "";



	switch (spectre_v2_user_stibp) {