Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e9b911a9 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Greg Kroah-Hartman
Browse files

powerpc/64s: Add support for a store forwarding barrier at kernel entry/exit 



commit a048a07d7f4535baa4cbad6bc024f175317ab938 upstream.

On some CPUs we can prevent a vulnerability related to store-to-load
forwarding by preventing store forwarding between privilege domains,
by inserting a barrier in kernel entry and exit paths.

This is known to be the case on at least Power7, Power8 and Power9
powerpc CPUs.

Barriers must be inserted generally before the first load after moving
to a higher privilege, and after the last store before moving to a
lower privilege, HV and PR privilege transitions must be protected.

Barriers are added as patch sections, with all kernel/hypervisor entry
points patched, and the exit points to lower privilge levels patched
similarly to the RFI flush patching.

Firmware advertisement is not implemented yet, so CPU flush types
are hard coded.

Thanks to Michal Suchánek for bug fixes and review.

Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMauricio Faria de Oliveira <mauricfo@linux.vnet.ibm.com>
Signed-off-by: default avatarMichael Neuling <mikey@neuling.org>
Signed-off-by: default avatarMichal Suchánek <msuchanek@suse.de>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent 2149936d

arch/powerpc/include/asm/exception-64s.h

+29 −0
Original line number Diff line number Diff line
@@ -51,6 +51,27 @@ 
#define EX_PPR		88	/* SMT thread status register (priority) */

#define EX_CTR		96



#define STF_ENTRY_BARRIER_SLOT						\

	STF_ENTRY_BARRIER_FIXUP_SECTION;				\

	nop;								\

	nop;								\

	nop



#define STF_EXIT_BARRIER_SLOT						\

	STF_EXIT_BARRIER_FIXUP_SECTION;					\

	nop;								\

	nop;								\

	nop;								\

	nop;								\

	nop;								\

	nop



/*

 * r10 must be free to use, r13 must be paca

 */

#define INTERRUPT_TO_KERNEL						\

	STF_ENTRY_BARRIER_SLOT



/*

 * Macros for annotating the expected destination of (h)rfid

 *
@@ -67,16 +88,19 @@ 
	rfid



#define RFI_TO_USER							\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	rfid;								\

	b	rfi_flush_fallback



#define RFI_TO_USER_OR_KERNEL						\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	rfid;								\

	b	rfi_flush_fallback



#define RFI_TO_GUEST							\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	rfid;								\

	b	rfi_flush_fallback
@@ -85,21 +109,25 @@ 
	hrfid



#define HRFI_TO_USER							\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	hrfid;								\

	b	hrfi_flush_fallback



#define HRFI_TO_USER_OR_KERNEL						\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	hrfid;								\

	b	hrfi_flush_fallback



#define HRFI_TO_GUEST							\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	hrfid;								\

	b	hrfi_flush_fallback



#define HRFI_TO_UNKNOWN							\

	STF_EXIT_BARRIER_SLOT;						\

	RFI_FLUSH_SLOT;							\

	hrfid;								\

	b	hrfi_flush_fallback
@@ -225,6 +253,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) 
#define __EXCEPTION_PROLOG_1(area, extra, vec)				\

	OPT_SAVE_REG_TO_PACA(area+EX_PPR, r9, CPU_FTR_HAS_PPR);		\

	OPT_SAVE_REG_TO_PACA(area+EX_CFAR, r10, CPU_FTR_CFAR);		\

	INTERRUPT_TO_KERNEL;						\

	SAVE_CTR(r10, area);						\

	mfcr	r9;							\

	extra(vec);							\

arch/powerpc/include/asm/feature-fixups.h

+19 −0
Original line number Diff line number Diff line
@@ -189,6 +189,22 @@ void apply_feature_fixups(void); 
void setup_feature_keys(void);

#endif



#define STF_ENTRY_BARRIER_FIXUP_SECTION			\

953:							\

	.pushsection __stf_entry_barrier_fixup,"a";	\

	.align 2;					\

954:							\

	FTR_ENTRY_OFFSET 953b-954b;			\

	.popsection;



#define STF_EXIT_BARRIER_FIXUP_SECTION			\

955:							\

	.pushsection __stf_exit_barrier_fixup,"a";	\

	.align 2;					\

956:							\

	FTR_ENTRY_OFFSET 955b-956b;			\

	.popsection;



#define RFI_FLUSH_FIXUP_SECTION				\

951:							\

	.pushsection __rfi_flush_fixup,"a";		\
@@ -200,6 +216,9 @@ void setup_feature_keys(void); 


#ifndef __ASSEMBLY__



extern long stf_barrier_fallback;

extern long __start___stf_entry_barrier_fixup, __stop___stf_entry_barrier_fixup;

extern long __start___stf_exit_barrier_fixup, __stop___stf_exit_barrier_fixup;

extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup;



#endif

arch/powerpc/include/asm/security_features.h

+11 −0
Original line number Diff line number Diff line
@@ -12,6 +12,17 @@ 
extern unsigned long powerpc_security_features;

extern bool rfi_flush;



/* These are bit flags */

enum stf_barrier_type {

	STF_BARRIER_NONE	= 0x1,

	STF_BARRIER_FALLBACK	= 0x2,

	STF_BARRIER_EIEIO	= 0x4,

	STF_BARRIER_SYNC_ORI	= 0x8,

};



void setup_stf_barrier(void);

void do_stf_barrier_fixups(enum stf_barrier_type types);



static inline void security_ftr_set(unsigned long feature)

{

	powerpc_security_features |= feature;

arch/powerpc/kernel/exceptions-64s.S

+15 −1
Original line number Diff line number Diff line
@@ -846,7 +846,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) 
#endif





EXC_REAL_MASKABLE(decrementer, 0x900, 0x980)

EXC_REAL_OOL_MASKABLE(decrementer, 0x900, 0x980)

EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x4980, 0x900)

TRAMP_KVM(PACA_EXGEN, 0x900)

EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt)
@@ -884,6 +884,7 @@ BEGIN_FTR_SECTION \ 
END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)				\

	mr	r9,r13 ;					\

	GET_PACA(r13) ;						\

	INTERRUPT_TO_KERNEL ;					\

	mfspr	r11,SPRN_SRR0 ;					\

0:
@@ -1353,6 +1354,19 @@ masked_##_H##interrupt: \ 
	##_H##RFI_TO_KERNEL;				\

	b	.



TRAMP_REAL_BEGIN(stf_barrier_fallback)

	std	r9,PACA_EXRFI+EX_R9(r13)

	std	r10,PACA_EXRFI+EX_R10(r13)

	sync

	ld	r9,PACA_EXRFI+EX_R9(r13)

	ld	r10,PACA_EXRFI+EX_R10(r13)

	ori	31,31,0

	.rept 14

	b	1f

1:

	.endr

	blr



/*

 * Real mode exceptions actually use this too, but alternate

 * instruction code patches (which end up in the common .text area)

arch/powerpc/kernel/security.c

+149 −0
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
// Copyright 2018, Michael Ellerman, IBM Corporation.



#include <linux/kernel.h>

#include <linux/debugfs.h>

#include <linux/device.h>

#include <linux/seq_buf.h>
@@ -86,3 +87,151 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, c 


	return s.len;

}



/*

 * Store-forwarding barrier support.

 */



static enum stf_barrier_type stf_enabled_flush_types;

static bool no_stf_barrier;

bool stf_barrier;



static int __init handle_no_stf_barrier(char *p)

{

	pr_info("stf-barrier: disabled on command line.");

	no_stf_barrier = true;

	return 0;

}



early_param("no_stf_barrier", handle_no_stf_barrier);



/* This is the generic flag used by other architectures */

static int __init handle_ssbd(char *p)

{

	if (!p || strncmp(p, "auto", 5) == 0 || strncmp(p, "on", 2) == 0 ) {

		/* Until firmware tells us, we have the barrier with auto */

		return 0;

	} else if (strncmp(p, "off", 3) == 0) {

		handle_no_stf_barrier(NULL);

		return 0;

	} else

		return 1;



	return 0;

}

early_param("spec_store_bypass_disable", handle_ssbd);



/* This is the generic flag used by other architectures */

static int __init handle_no_ssbd(char *p)

{

	handle_no_stf_barrier(NULL);

	return 0;

}

early_param("nospec_store_bypass_disable", handle_no_ssbd);



static void stf_barrier_enable(bool enable)

{

	if (enable)

		do_stf_barrier_fixups(stf_enabled_flush_types);

	else

		do_stf_barrier_fixups(STF_BARRIER_NONE);



	stf_barrier = enable;

}



void setup_stf_barrier(void)

{

	enum stf_barrier_type type;

	bool enable, hv;



	hv = cpu_has_feature(CPU_FTR_HVMODE);



	/* Default to fallback in case fw-features are not available */

	if (cpu_has_feature(CPU_FTR_ARCH_300))

		type = STF_BARRIER_EIEIO;

	else if (cpu_has_feature(CPU_FTR_ARCH_207S))

		type = STF_BARRIER_SYNC_ORI;

	else if (cpu_has_feature(CPU_FTR_ARCH_206))

		type = STF_BARRIER_FALLBACK;

	else

		type = STF_BARRIER_NONE;



	enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&

		(security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR) ||

		 (security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV) && hv));



	if (type == STF_BARRIER_FALLBACK) {

		pr_info("stf-barrier: fallback barrier available\n");

	} else if (type == STF_BARRIER_SYNC_ORI) {

		pr_info("stf-barrier: hwsync barrier available\n");

	} else if (type == STF_BARRIER_EIEIO) {

		pr_info("stf-barrier: eieio barrier available\n");

	}



	stf_enabled_flush_types = type;



	if (!no_stf_barrier)

		stf_barrier_enable(enable);

}



ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)

{

	if (stf_barrier && stf_enabled_flush_types != STF_BARRIER_NONE) {

		const char *type;

		switch (stf_enabled_flush_types) {

		case STF_BARRIER_EIEIO:

			type = "eieio";

			break;

		case STF_BARRIER_SYNC_ORI:

			type = "hwsync";

			break;

		case STF_BARRIER_FALLBACK:

			type = "fallback";

			break;

		default:

			type = "unknown";

		}

		return sprintf(buf, "Mitigation: Kernel entry/exit barrier (%s)\n", type);

	}



	if (!security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV) &&

	    !security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR))

		return sprintf(buf, "Not affected\n");



	return sprintf(buf, "Vulnerable\n");

}



#ifdef CONFIG_DEBUG_FS

static int stf_barrier_set(void *data, u64 val)

{

	bool enable;



	if (val == 1)

		enable = true;

	else if (val == 0)

		enable = false;

	else

		return -EINVAL;



	/* Only do anything if we're changing state */

	if (enable != stf_barrier)

		stf_barrier_enable(enable);



	return 0;

}



static int stf_barrier_get(void *data, u64 *val)

{

	*val = stf_barrier ? 1 : 0;

	return 0;

}



DEFINE_SIMPLE_ATTRIBUTE(fops_stf_barrier, stf_barrier_get, stf_barrier_set, "%llu\n");



static __init int stf_barrier_debugfs_init(void)

{

	debugfs_create_file("stf_barrier", 0600, powerpc_debugfs_root, NULL, &fops_stf_barrier);

	return 0;

}

device_initcall(stf_barrier_debugfs_init);

#endif /* CONFIG_DEBUG_FS */