Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2cdc2e62 authored by Christoffer Dall's avatar Christoffer Dall Committed by Greg Kroah-Hartman
Browse files

KVM: arm64: Avoid storing the vcpu pointer on the stack 



Commit 4464e210de9e80e38de59df052fe09ea2ff80b1b upstream.

We already have the percpu area for the host cpu state, which points to
the VCPU, so there's no need to store the VCPU pointer on the stack on
every context switch.  We can be a little more clever and just use
tpidr_el2 for the percpu offset and load the VCPU pointer from the host
context.

This has the benefit of being able to retrieve the host context even
when our stack is corrupted, and it has a potential performance benefit
because we trade a store plus a load for an mrs and a load on a round
trip to the guest.

This does require us to calculate the percpu offset without including
the offset from the kernel mapping of the percpu array to the linear
mapping of the array (which is what we store in tpidr_el1), because a
PC-relative generated address in EL2 is already giving us the hyp alias
of the linear mapping of a kernel address.  We do this in
__cpu_init_hyp_mode() by using kvm_ksym_ref().

The code that accesses ESR_EL2 was previously using an alternative to
use the _EL1 accessor on VHE systems, but this was actually unnecessary
as the _EL1 accessor aliases the ESR_EL2 register on VHE, and the _EL2
accessor does the same thing on both systems.

Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
Reviewed-by: default avatarAndrew Jones <drjones@redhat.com>
Signed-off-by: default avatarChristoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent dca78156

arch/arm64/include/asm/kvm_asm.h

+15 −0
Original line number Original line Diff line number Diff line
@@ -33,6 +33,7 @@ 
#define KVM_ARM64_DEBUG_DIRTY_SHIFT	0

#define KVM_ARM64_DEBUG_DIRTY_SHIFT	0

#define KVM_ARM64_DEBUG_DIRTY		(1 << KVM_ARM64_DEBUG_DIRTY_SHIFT)

#define KVM_ARM64_DEBUG_DIRTY		(1 << KVM_ARM64_DEBUG_DIRTY_SHIFT)





/* Translate a kernel address of @sym into its equivalent linear mapping */

#define kvm_ksym_ref(sym)						\

#define kvm_ksym_ref(sym)						\

	({								\

	({								\

		void *val = &sym;					\

		void *val = &sym;					\
@@ -68,6 +69,20 @@ extern u32 __init_stage2_translation(void); 




extern void __qcom_hyp_sanitize_btac_predictors(void);

extern void __qcom_hyp_sanitize_btac_predictors(void);





#else /* __ASSEMBLY__ */



.macro get_host_ctxt reg, tmp

	adr_l	\reg, kvm_host_cpu_state

	mrs	\tmp, tpidr_el2

	add	\reg, \reg, \tmp

.endm



.macro get_vcpu_ptr vcpu, ctxt

	get_host_ctxt \ctxt, \vcpu

	ldr	\vcpu, [\ctxt, #HOST_CONTEXT_VCPU]

	kern_hyp_va	\vcpu

.endm



#endif

#endif





#endif /* __ARM_KVM_ASM_H__ */
 
#endif /* __ARM_KVM_ASM_H__ */

arch/arm64/include/asm/kvm_host.h

+15 −0
Original line number Original line Diff line number Diff line
@@ -350,10 +350,15 @@ int kvm_perf_teardown(void); 




struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);

struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);





void __kvm_set_tpidr_el2(u64 tpidr_el2);

DECLARE_PER_CPU(kvm_cpu_context_t, kvm_host_cpu_state);



static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,

static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,

				       unsigned long hyp_stack_ptr,

				       unsigned long hyp_stack_ptr,

				       unsigned long vector_ptr)

				       unsigned long vector_ptr)

{

{

	u64 tpidr_el2;



	/*

	/*

	 * Call initialization code, and switch to the full blown HYP code.

	 * Call initialization code, and switch to the full blown HYP code.

	 * If the cpucaps haven't been finalized yet, something has gone very

	 * If the cpucaps haven't been finalized yet, something has gone very
@@ -362,6 +367,16 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, 
	 */

	 */

	BUG_ON(!static_branch_likely(&arm64_const_caps_ready));

	BUG_ON(!static_branch_likely(&arm64_const_caps_ready));

	__kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);

	__kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);



	/*

	 * Calculate the raw per-cpu offset without a translation from the

	 * kernel's mapping to the linear mapping, and store it in tpidr_el2

	 * so that we can use adr_l to access per-cpu variables in EL2.

	 */

	tpidr_el2 = (u64)this_cpu_ptr(&kvm_host_cpu_state)

		- (u64)kvm_ksym_ref(kvm_host_cpu_state);



	kvm_call_hyp(__kvm_set_tpidr_el2, tpidr_el2);

}

}





static inline void kvm_arch_hardware_unsetup(void) {}

static inline void kvm_arch_hardware_unsetup(void) {}

arch/arm64/kernel/asm-offsets.c

+1 −0
Original line number Original line Diff line number Diff line
@@ -136,6 +136,7 @@ int main(void) 
  DEFINE(CPU_FP_REGS,		offsetof(struct kvm_regs, fp_regs));

  DEFINE(CPU_FP_REGS,		offsetof(struct kvm_regs, fp_regs));

  DEFINE(VCPU_FPEXC32_EL2,	offsetof(struct kvm_vcpu, arch.ctxt.sys_regs[FPEXC32_EL2]));

  DEFINE(VCPU_FPEXC32_EL2,	offsetof(struct kvm_vcpu, arch.ctxt.sys_regs[FPEXC32_EL2]));

  DEFINE(VCPU_HOST_CONTEXT,	offsetof(struct kvm_vcpu, arch.host_cpu_context));

  DEFINE(VCPU_HOST_CONTEXT,	offsetof(struct kvm_vcpu, arch.host_cpu_context));

  DEFINE(HOST_CONTEXT_VCPU,	offsetof(struct kvm_cpu_context, __hyp_running_vcpu));

#endif

#endif

#ifdef CONFIG_CPU_PM

#ifdef CONFIG_CPU_PM

  DEFINE(CPU_SUSPEND_SZ,	sizeof(struct cpu_suspend_ctx));

  DEFINE(CPU_SUSPEND_SZ,	sizeof(struct cpu_suspend_ctx));

arch/arm64/kvm/hyp/entry.S

+1 −5
Original line number Original line Diff line number Diff line
@@ -62,9 +62,6 @@ ENTRY(__guest_enter) 
	// Store the host regs

	// Store the host regs

	save_callee_saved_regs x1

	save_callee_saved_regs x1





	// Store host_ctxt and vcpu for use at exit time

	stp	x1, x0, [sp, #-16]!



	add	x18, x0, #VCPU_CONTEXT

	add	x18, x0, #VCPU_CONTEXT





	// Restore guest regs x0-x17

	// Restore guest regs x0-x17
@@ -118,8 +115,7 @@ ENTRY(__guest_exit) 
	// Store the guest regs x19-x29, lr

	// Store the guest regs x19-x29, lr

	save_callee_saved_regs x1

	save_callee_saved_regs x1





	// Restore the host_ctxt from the stack

	get_host_ctxt	x2, x3

	ldr	x2, [sp], #16





	// Now restore the host regs

	// Now restore the host regs

	restore_callee_saved_regs x2

	restore_callee_saved_regs x2

arch/arm64/kvm/hyp/hyp-entry.S

+10 −18
Original line number Original line Diff line number Diff line
@@ -57,13 +57,8 @@ ENDPROC(__vhe_hyp_call) 
el1_sync:				// Guest trapped into EL2

el1_sync:				// Guest trapped into EL2

	stp	x0, x1, [sp, #-16]!

	stp	x0, x1, [sp, #-16]!





alternative_if_not ARM64_HAS_VIRT_HOST_EXTN

	mrs	x0, esr_el2

	mrs	x1, esr_el2

	lsr	x0, x0, #ESR_ELx_EC_SHIFT

alternative_else

	mrs	x1, esr_el1

alternative_endif

	lsr	x0, x1, #ESR_ELx_EC_SHIFT



	cmp	x0, #ESR_ELx_EC_HVC64

	cmp	x0, #ESR_ELx_EC_HVC64

	ccmp	x0, #ESR_ELx_EC_HVC32, #4, ne

	ccmp	x0, #ESR_ELx_EC_HVC32, #4, ne

	b.ne	el1_trap

	b.ne	el1_trap
@@ -117,10 +112,14 @@ el1_hvc_guest: 
	eret

	eret





el1_trap:

el1_trap:

	get_vcpu_ptr	x1, x0



	mrs		x0, esr_el2

	lsr		x0, x0, #ESR_ELx_EC_SHIFT

	/*

	/*

	 * x0: ESR_EC

	 * x0: ESR_EC

	 * x1: vcpu pointer

	 */

	 */

	ldr	x1, [sp, #16 + 8]	// vcpu stored by __guest_enter





	/*

	/*

	 * We trap the first access to the FP/SIMD to save the host context

	 * We trap the first access to the FP/SIMD to save the host context
@@ -138,13 +137,13 @@ alternative_else_nop_endif 




el1_irq:

el1_irq:

	stp     x0, x1, [sp, #-16]!

	stp     x0, x1, [sp, #-16]!

	ldr	x1, [sp, #16 + 8]

	get_vcpu_ptr	x1, x0

	mov	x0, #ARM_EXCEPTION_IRQ

	mov	x0, #ARM_EXCEPTION_IRQ

	b	__guest_exit

	b	__guest_exit





el1_error:

el1_error:

	stp     x0, x1, [sp, #-16]!

	stp     x0, x1, [sp, #-16]!

	ldr	x1, [sp, #16 + 8]

	get_vcpu_ptr	x1, x0

	mov	x0, #ARM_EXCEPTION_EL1_SERROR

	mov	x0, #ARM_EXCEPTION_EL1_SERROR

	b	__guest_exit

	b	__guest_exit
@@ -180,14 +179,7 @@ ENTRY(__hyp_do_panic) 
ENDPROC(__hyp_do_panic)

ENDPROC(__hyp_do_panic)





ENTRY(__hyp_panic)

ENTRY(__hyp_panic)

	/*

	get_host_ctxt x0, x1

	 * '=kvm_host_cpu_state' is a host VA from the constant pool, it may

	 * not be accessible by this address from EL2, hyp_panic() converts

	 * it with kern_hyp_va() before use.

	 */

	ldr	x0, =kvm_host_cpu_state

	mrs	x1, tpidr_el2

	add	x0, x0, x1

	b	hyp_panic

	b	hyp_panic

ENDPROC(__hyp_panic)

ENDPROC(__hyp_panic)