Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c45d885c authored by James Morse's avatar James Morse Committed by Greg Kroah-Hartman
Browse files

arm64: Add percpu vectors for EL1 



commit bd09128d16fac3c34b80bd6a29088ac632e8ce09 upstream.

The Spectre-BHB workaround adds a firmware call to the vectors. This
is needed on some CPUs, but not others. To avoid the unaffected CPU in
a big/little pair from making the firmware call, create per cpu vectors.

The per-cpu vectors only apply when returning from EL0.

Systems using KPTI can use the canonical 'full-fat' vectors directly at
EL1, the trampoline exit code will switch to this_cpu_vector on exit to
EL0. Systems not using KPTI should always use this_cpu_vector.

this_cpu_vector will point at a vector in tramp_vecs or
__bp_harden_el1_vectors, depending on whether KPTI is in use.

Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Signed-off-by: default avatarJames Morse <james.morse@arm.com>
Signed-off-by: default avatarSasha Levin <sashal@kernel.org>
parent 1bb19449

arch/arm64/include/asm/mmu.h

+1 −1
Original line number Diff line number Diff line
@@ -29,7 +29,7 @@ typedef struct { 
 */

#define ASID(mm)	((mm)->context.id.counter & 0xffff)



static inline bool arm64_kernel_unmapped_at_el0(void)

static __always_inline bool arm64_kernel_unmapped_at_el0(void)

{

	return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) &&

	       cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0);

arch/arm64/include/asm/vectors.h

+27 −0
Original line number Diff line number Diff line
@@ -5,6 +5,15 @@ 
#ifndef __ASM_VECTORS_H

#define __ASM_VECTORS_H



#include <linux/bug.h>

#include <linux/percpu.h>



#include <asm/fixmap.h>



extern char vectors[];

extern char tramp_vectors[];

extern char __bp_harden_el1_vectors[];



/*

 * Note: the order of this enum corresponds to two arrays in entry.S:

 * tramp_vecs and __bp_harden_el1_vectors. By default the canonical
@@ -31,4 +40,22 @@ enum arm64_bp_harden_el1_vectors { 
	EL1_VECTOR_KPTI,

};



/* The vectors to use on return from EL0. e.g. to remap the kernel */

DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector);



#ifndef CONFIG_UNMAP_KERNEL_AT_EL0

#define TRAMP_VALIAS	0

#endif



static inline const char *

arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot)

{

	if (arm64_kernel_unmapped_at_el0())

		return (char *)TRAMP_VALIAS + SZ_2K * slot;



	WARN_ON_ONCE(slot == EL1_VECTOR_KPTI);



	return __bp_harden_el1_vectors + SZ_2K * slot;

}



#endif /* __ASM_VECTORS_H */

arch/arm64/kernel/cpufeature.c

+11 −0
Original line number Diff line number Diff line
@@ -10,11 +10,13 @@ 
#include <linux/bsearch.h>

#include <linux/cpumask.h>

#include <linux/crash_dump.h>

#include <linux/percpu.h>

#include <linux/sort.h>

#include <linux/stop_machine.h>

#include <linux/types.h>

#include <linux/mm.h>

#include <linux/cpu.h>



#include <asm/cpu.h>

#include <asm/cpufeature.h>

#include <asm/cpu_ops.h>
@@ -23,6 +25,7 @@ 
#include <asm/processor.h>

#include <asm/sysreg.h>

#include <asm/traps.h>

#include <asm/vectors.h>

#include <asm/virt.h>



/* Kernel representation of AT_HWCAP and AT_HWCAP2 */
@@ -45,6 +48,8 @@ static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM6 
/* Need also bit for ARM64_CB_PATCH */

DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);



DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;



/*

 * Flag to indicate if we have computed the system wide

 * capabilities based on the boot time active CPUs. This
@@ -1047,6 +1052,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) 
	static bool kpti_applied = false;

	int cpu = smp_processor_id();



	if (__this_cpu_read(this_cpu_vector) == vectors) {

		const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);



		__this_cpu_write(this_cpu_vector, v);

	}



	/*

	 * We don't need to rewrite the page-tables if either we've done

	 * it already or we have KASLR enabled and therefore have not

arch/arm64/kernel/entry.S

+10 −6
Original line number Diff line number Diff line
@@ -60,7 +60,6 @@ 
	.macro kernel_ventry, el, label, regsize = 64

	.align 7

.Lventry_start\@:

#ifdef CONFIG_UNMAP_KERNEL_AT_EL0

	.if	\el == 0

	/*

	 * This must be the first instruction of the EL0 vector entries. It is
@@ -75,7 +74,6 @@ 
	.endif

.Lskip_tramp_vectors_cleanup\@:

	.endif

#endif



	sub	sp, sp, #S_FRAME_SIZE

#ifdef CONFIG_VMAP_STACK
@@ -1129,10 +1127,14 @@ alternative_else_nop_endif 
	.endm



	.macro tramp_exit, regsize = 64

	adr	x30, tramp_vectors

#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY

	add	x30, x30, SZ_4K

#endif

	tramp_data_read_var	x30, this_cpu_vector

alternative_if_not ARM64_HAS_VIRT_HOST_EXTN

	mrs	x29, tpidr_el1

alternative_else

	mrs	x29, tpidr_el2

alternative_endif

	ldr	x30, [x30, x29]



	msr	vbar_el1, x30

	ldr	lr, [sp, #S_LR]

	tramp_unmap_kernel	x29
@@ -1193,6 +1195,8 @@ __entry_tramp_data_vectors: 
__entry_tramp_data___sdei_asm_trampoline_next_handler:

	.quad	__sdei_asm_handler

#endif /* CONFIG_ARM_SDE_INTERFACE */

__entry_tramp_data_this_cpu_vector:

	.quad	this_cpu_vector

	.popsection				// .rodata

#endif /* CONFIG_RANDOMIZE_BASE */

#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */

arch/arm64/kvm/hyp/switch.c

+6 −2
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include <asm/debug-monitors.h>

#include <asm/processor.h>

#include <asm/thread_info.h>

#include <asm/vectors.h>



extern struct exception_table_entry __start___kvm_ex_table;

extern struct exception_table_entry __stop___kvm_ex_table;
@@ -152,7 +153,7 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu) 


static void deactivate_traps_vhe(void)

{

	extern char vectors[];	/* kernel exception vectors */

	const char *host_vectors = vectors;

	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);



	/*
@@ -163,7 +164,10 @@ static void deactivate_traps_vhe(void) 
	asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_1165522));



	write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);

	write_sysreg(vectors, vbar_el1);



	if (!arm64_kernel_unmapped_at_el0())

		host_vectors = __this_cpu_read(this_cpu_vector);

	write_sysreg(host_vectors, vbar_el1);

}

NOKPROBE_SYMBOL(deactivate_traps_vhe);