Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1ea66d27 authored by Marc Zyngier's avatar Marc Zyngier
Browse files

arm64: KVM: Move away from the assembly version of the world switch 



This is it. We remove all of the code that has now been rewritten.

Signed-off-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
Acked-by: default avatarChristoffer Dall <christoffer.dall@linaro.org>
parent 910917bb

arch/arm64/kvm/Makefile

+0 −2
Original line number Diff line number Diff line
@@ -23,8 +23,6 @@ kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generi 
kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic.o

kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2.o

kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2-emul.o

kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v2-switch.o

kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o

kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o

kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o

kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o

arch/arm64/kvm/hyp.S

+1 −1080
Original line number Diff line number Diff line
@@ -17,910 +17,7 @@ 


#include <linux/linkage.h>



#include <asm/alternative.h>

#include <asm/asm-offsets.h>

#include <asm/assembler.h>

#include <asm/cpufeature.h>

#include <asm/debug-monitors.h>

#include <asm/esr.h>

#include <asm/fpsimdmacros.h>

#include <asm/kvm.h>

#include <asm/kvm_arm.h>

#include <asm/kvm_asm.h>

#include <asm/kvm_mmu.h>

#include <asm/memory.h>



#define CPU_GP_REG_OFFSET(x)	(CPU_GP_REGS + x)

#define CPU_XREG_OFFSET(x)	CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)

#define CPU_SPSR_OFFSET(x)	CPU_GP_REG_OFFSET(CPU_SPSR + 8*x)

#define CPU_SYSREG_OFFSET(x)	(CPU_SYSREGS + 8*x)



	.text

	.pushsection	.hyp.text, "ax"

	.align	PAGE_SHIFT



.macro save_common_regs

	// x2: base address for cpu context

	// x3: tmp register



	add	x3, x2, #CPU_XREG_OFFSET(19)

	stp	x19, x20, [x3]

	stp	x21, x22, [x3, #16]

	stp	x23, x24, [x3, #32]

	stp	x25, x26, [x3, #48]

	stp	x27, x28, [x3, #64]

	stp	x29, lr, [x3, #80]



	mrs	x19, sp_el0

	mrs	x20, elr_el2		// pc before entering el2

	mrs	x21, spsr_el2		// pstate before entering el2



	stp	x19, x20, [x3, #96]

	str	x21, [x3, #112]



	mrs	x22, sp_el1

	mrs	x23, elr_el1

	mrs	x24, spsr_el1



	str	x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]

	str	x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]

	str	x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]

.endm



.macro restore_common_regs

	// x2: base address for cpu context

	// x3: tmp register



	ldr	x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]

	ldr	x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]

	ldr	x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]



	msr	sp_el1, x22

	msr	elr_el1, x23

	msr	spsr_el1, x24



	add	x3, x2, #CPU_XREG_OFFSET(31)    // SP_EL0

	ldp	x19, x20, [x3]

	ldr	x21, [x3, #16]



	msr	sp_el0, x19

	msr	elr_el2, x20 		// pc on return from el2

	msr	spsr_el2, x21 		// pstate on return from el2



	add	x3, x2, #CPU_XREG_OFFSET(19)

	ldp	x19, x20, [x3]

	ldp	x21, x22, [x3, #16]

	ldp	x23, x24, [x3, #32]

	ldp	x25, x26, [x3, #48]

	ldp	x27, x28, [x3, #64]

	ldp	x29, lr, [x3, #80]

.endm



.macro save_host_regs

	save_common_regs

.endm



.macro restore_host_regs

	restore_common_regs

.endm



.macro save_fpsimd

	// x2: cpu context address

	// x3, x4: tmp regs

	add	x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)

	fpsimd_save x3, 4

.endm



.macro restore_fpsimd

	// x2: cpu context address

	// x3, x4: tmp regs

	add	x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)

	fpsimd_restore x3, 4

.endm



.macro save_guest_regs

	// x0 is the vcpu address

	// x1 is the return code, do not corrupt!

	// x2 is the cpu context

	// x3 is a tmp register

	// Guest's x0-x3 are on the stack



	// Compute base to save registers

	add	x3, x2, #CPU_XREG_OFFSET(4)

	stp	x4, x5, [x3]

	stp	x6, x7, [x3, #16]

	stp	x8, x9, [x3, #32]

	stp	x10, x11, [x3, #48]

	stp	x12, x13, [x3, #64]

	stp	x14, x15, [x3, #80]

	stp	x16, x17, [x3, #96]

	str	x18, [x3, #112]



	pop	x6, x7			// x2, x3

	pop	x4, x5			// x0, x1



	add	x3, x2, #CPU_XREG_OFFSET(0)

	stp	x4, x5, [x3]

	stp	x6, x7, [x3, #16]



	save_common_regs

.endm



.macro restore_guest_regs

	// x0 is the vcpu address.

	// x2 is the cpu context

	// x3 is a tmp register



	// Prepare x0-x3 for later restore

	add	x3, x2, #CPU_XREG_OFFSET(0)

	ldp	x4, x5, [x3]

	ldp	x6, x7, [x3, #16]

	push	x4, x5		// Push x0-x3 on the stack

	push	x6, x7



	// x4-x18

	ldp	x4, x5, [x3, #32]

	ldp	x6, x7, [x3, #48]

	ldp	x8, x9, [x3, #64]

	ldp	x10, x11, [x3, #80]

	ldp	x12, x13, [x3, #96]

	ldp	x14, x15, [x3, #112]

	ldp	x16, x17, [x3, #128]

	ldr	x18, [x3, #144]



	// x19-x29, lr, sp*, elr*, spsr*

	restore_common_regs



	// Last bits of the 64bit state

	pop	x2, x3

	pop	x0, x1



	// Do not touch any register after this!

.endm



/*

 * Macros to perform system register save/restore.

 *

 * Ordering here is absolutely critical, and must be kept consistent

 * in {save,restore}_sysregs, {save,restore}_guest_32bit_state,

 * and in kvm_asm.h.

 *

 * In other words, don't touch any of these unless you know what

 * you are doing.

 */

.macro save_sysregs

	// x2: base address for cpu context

	// x3: tmp register



	add	x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)



	mrs	x4,	vmpidr_el2

	mrs	x5,	csselr_el1

	mrs	x6,	sctlr_el1

	mrs	x7,	actlr_el1

	mrs	x8,	cpacr_el1

	mrs	x9,	ttbr0_el1

	mrs	x10,	ttbr1_el1

	mrs	x11,	tcr_el1

	mrs	x12,	esr_el1

	mrs	x13, 	afsr0_el1

	mrs	x14,	afsr1_el1

	mrs	x15,	far_el1

	mrs	x16,	mair_el1

	mrs	x17,	vbar_el1

	mrs	x18,	contextidr_el1

	mrs	x19,	tpidr_el0

	mrs	x20,	tpidrro_el0

	mrs	x21,	tpidr_el1

	mrs	x22, 	amair_el1

	mrs	x23, 	cntkctl_el1

	mrs	x24,	par_el1

	mrs	x25,	mdscr_el1



	stp	x4, x5, [x3]

	stp	x6, x7, [x3, #16]

	stp	x8, x9, [x3, #32]

	stp	x10, x11, [x3, #48]

	stp	x12, x13, [x3, #64]

	stp	x14, x15, [x3, #80]

	stp	x16, x17, [x3, #96]

	stp	x18, x19, [x3, #112]

	stp	x20, x21, [x3, #128]

	stp	x22, x23, [x3, #144]

	stp	x24, x25, [x3, #160]

.endm



.macro save_debug type

	// x4: pointer to register set

	// x5: number of registers to skip

	// x6..x22 trashed



	adr	x22, 1f

	add	x22, x22, x5, lsl #2

	br	x22

1:

	mrs	x21, \type\()15_el1

	mrs	x20, \type\()14_el1

	mrs	x19, \type\()13_el1

	mrs	x18, \type\()12_el1

	mrs	x17, \type\()11_el1

	mrs	x16, \type\()10_el1

	mrs	x15, \type\()9_el1

	mrs	x14, \type\()8_el1

	mrs	x13, \type\()7_el1

	mrs	x12, \type\()6_el1

	mrs	x11, \type\()5_el1

	mrs	x10, \type\()4_el1

	mrs	x9, \type\()3_el1

	mrs	x8, \type\()2_el1

	mrs	x7, \type\()1_el1

	mrs	x6, \type\()0_el1



	adr	x22, 1f

	add	x22, x22, x5, lsl #2

	br	x22

1:

	str	x21, [x4, #(15 * 8)]

	str	x20, [x4, #(14 * 8)]

	str	x19, [x4, #(13 * 8)]

	str	x18, [x4, #(12 * 8)]

	str	x17, [x4, #(11 * 8)]

	str	x16, [x4, #(10 * 8)]

	str	x15, [x4, #(9 * 8)]

	str	x14, [x4, #(8 * 8)]

	str	x13, [x4, #(7 * 8)]

	str	x12, [x4, #(6 * 8)]

	str	x11, [x4, #(5 * 8)]

	str	x10, [x4, #(4 * 8)]

	str	x9, [x4, #(3 * 8)]

	str	x8, [x4, #(2 * 8)]

	str	x7, [x4, #(1 * 8)]

	str	x6, [x4, #(0 * 8)]

.endm



.macro restore_sysregs

	// x2: base address for cpu context

	// x3: tmp register



	add	x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)



	ldp	x4, x5, [x3]

	ldp	x6, x7, [x3, #16]

	ldp	x8, x9, [x3, #32]

	ldp	x10, x11, [x3, #48]

	ldp	x12, x13, [x3, #64]

	ldp	x14, x15, [x3, #80]

	ldp	x16, x17, [x3, #96]

	ldp	x18, x19, [x3, #112]

	ldp	x20, x21, [x3, #128]

	ldp	x22, x23, [x3, #144]

	ldp	x24, x25, [x3, #160]



	msr	vmpidr_el2,	x4

	msr	csselr_el1,	x5

	msr	sctlr_el1,	x6

	msr	actlr_el1,	x7

	msr	cpacr_el1,	x8

	msr	ttbr0_el1,	x9

	msr	ttbr1_el1,	x10

	msr	tcr_el1,	x11

	msr	esr_el1,	x12

	msr	afsr0_el1,	x13

	msr	afsr1_el1,	x14

	msr	far_el1,	x15

	msr	mair_el1,	x16

	msr	vbar_el1,	x17

	msr	contextidr_el1,	x18

	msr	tpidr_el0,	x19

	msr	tpidrro_el0,	x20

	msr	tpidr_el1,	x21

	msr	amair_el1,	x22

	msr	cntkctl_el1,	x23

	msr	par_el1,	x24

	msr	mdscr_el1,	x25

.endm



.macro restore_debug type

	// x4: pointer to register set

	// x5: number of registers to skip

	// x6..x22 trashed



	adr	x22, 1f

	add	x22, x22, x5, lsl #2

	br	x22

1:

	ldr	x21, [x4, #(15 * 8)]

	ldr	x20, [x4, #(14 * 8)]

	ldr	x19, [x4, #(13 * 8)]

	ldr	x18, [x4, #(12 * 8)]

	ldr	x17, [x4, #(11 * 8)]

	ldr	x16, [x4, #(10 * 8)]

	ldr	x15, [x4, #(9 * 8)]

	ldr	x14, [x4, #(8 * 8)]

	ldr	x13, [x4, #(7 * 8)]

	ldr	x12, [x4, #(6 * 8)]

	ldr	x11, [x4, #(5 * 8)]

	ldr	x10, [x4, #(4 * 8)]

	ldr	x9, [x4, #(3 * 8)]

	ldr	x8, [x4, #(2 * 8)]

	ldr	x7, [x4, #(1 * 8)]

	ldr	x6, [x4, #(0 * 8)]



	adr	x22, 1f

	add	x22, x22, x5, lsl #2

	br	x22

1:

	msr	\type\()15_el1, x21

	msr	\type\()14_el1, x20

	msr	\type\()13_el1, x19

	msr	\type\()12_el1, x18

	msr	\type\()11_el1, x17

	msr	\type\()10_el1, x16

	msr	\type\()9_el1, x15

	msr	\type\()8_el1, x14

	msr	\type\()7_el1, x13

	msr	\type\()6_el1, x12

	msr	\type\()5_el1, x11

	msr	\type\()4_el1, x10

	msr	\type\()3_el1, x9

	msr	\type\()2_el1, x8

	msr	\type\()1_el1, x7

	msr	\type\()0_el1, x6

.endm



.macro skip_32bit_state tmp, target

	// Skip 32bit state if not needed

	mrs	\tmp, hcr_el2

	tbnz	\tmp, #HCR_RW_SHIFT, \target

.endm



.macro skip_tee_state tmp, target

	// Skip ThumbEE state if not needed

	mrs	\tmp, id_pfr0_el1

	tbz	\tmp, #12, \target

.endm



.macro skip_debug_state tmp, target

	ldr	\tmp, [x0, #VCPU_DEBUG_FLAGS]

	tbz	\tmp, #KVM_ARM64_DEBUG_DIRTY_SHIFT, \target

.endm



/*

 * Branch to target if CPTR_EL2.TFP bit is set (VFP/SIMD trapping enabled)

 */

.macro skip_fpsimd_state tmp, target

	mrs	\tmp, cptr_el2

	tbnz	\tmp, #CPTR_EL2_TFP_SHIFT, \target

.endm



.macro compute_debug_state target

	// Compute debug state: If any of KDE, MDE or KVM_ARM64_DEBUG_DIRTY

	// is set, we do a full save/restore cycle and disable trapping.

	add	x25, x0, #VCPU_CONTEXT



	// Check the state of MDSCR_EL1

	ldr	x25, [x25, #CPU_SYSREG_OFFSET(MDSCR_EL1)]

	and	x26, x25, #DBG_MDSCR_KDE

	and	x25, x25, #DBG_MDSCR_MDE

	adds	xzr, x25, x26

	b.eq	9998f		// Nothing to see there



	// If any interesting bits was set, we must set the flag

	mov	x26, #KVM_ARM64_DEBUG_DIRTY

	str	x26, [x0, #VCPU_DEBUG_FLAGS]

	b	9999f		// Don't skip restore



9998:

	// Otherwise load the flags from memory in case we recently

	// trapped

	skip_debug_state x25, \target

9999:

.endm



.macro save_guest_32bit_state

	skip_32bit_state x3, 1f



	add	x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT)

	mrs	x4, spsr_abt

	mrs	x5, spsr_und

	mrs	x6, spsr_irq

	mrs	x7, spsr_fiq

	stp	x4, x5, [x3]

	stp	x6, x7, [x3, #16]



	add	x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2)

	mrs	x4, dacr32_el2

	mrs	x5, ifsr32_el2

	stp	x4, x5, [x3]



	skip_fpsimd_state x8, 2f

	mrs	x6, fpexc32_el2

	str	x6, [x3, #16]

2:

	skip_debug_state x8, 1f

	mrs	x7, dbgvcr32_el2

	str	x7, [x3, #24]

1:

.endm



.macro restore_guest_32bit_state

	skip_32bit_state x3, 1f



	add	x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT)

	ldp	x4, x5, [x3]

	ldp	x6, x7, [x3, #16]

	msr	spsr_abt, x4

	msr	spsr_und, x5

	msr	spsr_irq, x6

	msr	spsr_fiq, x7



	add	x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2)

	ldp	x4, x5, [x3]

	msr	dacr32_el2, x4

	msr	ifsr32_el2, x5



	skip_debug_state x8, 1f

	ldr	x7, [x3, #24]

	msr	dbgvcr32_el2, x7

1:

.endm



.macro activate_traps

	ldr     x2, [x0, #VCPU_HCR_EL2]



	/*

	 * We are about to set CPTR_EL2.TFP to trap all floating point

	 * register accesses to EL2, however, the ARM ARM clearly states that

	 * traps are only taken to EL2 if the operation would not otherwise

	 * trap to EL1.  Therefore, always make sure that for 32-bit guests,

	 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.

	 */

	tbnz	x2, #HCR_RW_SHIFT, 99f // open code skip_32bit_state

	mov	x3, #(1 << 30)

	msr	fpexc32_el2, x3

	isb

99:

	msr     hcr_el2, x2

	mov	x2, #CPTR_EL2_TTA

	orr     x2, x2, #CPTR_EL2_TFP

	msr	cptr_el2, x2



	mov	x2, #(1 << 15)	// Trap CP15 Cr=15

	msr	hstr_el2, x2



	// Monitor Debug Config - see kvm_arm_setup_debug()

	ldr	x2, [x0, #VCPU_MDCR_EL2]

	msr	mdcr_el2, x2

.endm



.macro deactivate_traps

	mov	x2, #HCR_RW

	msr	hcr_el2, x2

	msr	hstr_el2, xzr



	mrs	x2, mdcr_el2

	and	x2, x2, #MDCR_EL2_HPMN_MASK

	msr	mdcr_el2, x2

.endm



.macro activate_vm

	ldr	x1, [x0, #VCPU_KVM]

	kern_hyp_va	x1

	ldr	x2, [x1, #KVM_VTTBR]

	msr	vttbr_el2, x2

.endm



.macro deactivate_vm

	msr	vttbr_el2, xzr

.endm



/*

 * Call into the vgic backend for state saving

 */

.macro save_vgic_state

alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF

	bl	__save_vgic_v2_state

alternative_else

	bl	__save_vgic_v3_state

alternative_endif

	mrs	x24, hcr_el2

	mov	x25, #HCR_INT_OVERRIDE

	neg	x25, x25

	and	x24, x24, x25

	msr	hcr_el2, x24

.endm



/*

 * Call into the vgic backend for state restoring

 */

.macro restore_vgic_state

	mrs	x24, hcr_el2

	ldr	x25, [x0, #VCPU_IRQ_LINES]

	orr	x24, x24, #HCR_INT_OVERRIDE

	orr	x24, x24, x25

	msr	hcr_el2, x24

alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF

	bl	__restore_vgic_v2_state

alternative_else

	bl	__restore_vgic_v3_state

alternative_endif

.endm



.macro save_timer_state

	// x0: vcpu pointer

	ldr	x2, [x0, #VCPU_KVM]

	kern_hyp_va x2

	ldr	w3, [x2, #KVM_TIMER_ENABLED]

	cbz	w3, 1f



	mrs	x3, cntv_ctl_el0

	and	x3, x3, #3

	str	w3, [x0, #VCPU_TIMER_CNTV_CTL]



	isb



	mrs	x3, cntv_cval_el0

	str	x3, [x0, #VCPU_TIMER_CNTV_CVAL]



1:

	// Disable the virtual timer

	msr	cntv_ctl_el0, xzr



	// Allow physical timer/counter access for the host

	mrs	x2, cnthctl_el2

	orr	x2, x2, #3

	msr	cnthctl_el2, x2



	// Clear cntvoff for the host

	msr	cntvoff_el2, xzr

.endm



.macro restore_timer_state

	// x0: vcpu pointer

	// Disallow physical timer access for the guest

	// Physical counter access is allowed

	mrs	x2, cnthctl_el2

	orr	x2, x2, #1

	bic	x2, x2, #2

	msr	cnthctl_el2, x2



	ldr	x2, [x0, #VCPU_KVM]

	kern_hyp_va x2

	ldr	w3, [x2, #KVM_TIMER_ENABLED]

	cbz	w3, 1f



	ldr	x3, [x2, #KVM_TIMER_CNTVOFF]

	msr	cntvoff_el2, x3

	ldr	x2, [x0, #VCPU_TIMER_CNTV_CVAL]

	msr	cntv_cval_el0, x2

	isb



	ldr	w2, [x0, #VCPU_TIMER_CNTV_CTL]

	and	x2, x2, #3

	msr	cntv_ctl_el0, x2

1:

.endm



__save_sysregs:

	save_sysregs

	ret



__restore_sysregs:

	restore_sysregs

	ret



/* Save debug state */

__save_debug:

	// x2: ptr to CPU context

	// x3: ptr to debug reg struct

	// x4/x5/x6-22/x24-26: trashed



	mrs	x26, id_aa64dfr0_el1

	ubfx	x24, x26, #12, #4	// Extract BRPs

	ubfx	x25, x26, #20, #4	// Extract WRPs

	mov	w26, #15

	sub	w24, w26, w24		// How many BPs to skip

	sub	w25, w26, w25		// How many WPs to skip



	mov	x5, x24

	add	x4, x3, #DEBUG_BCR

	save_debug dbgbcr

	add	x4, x3, #DEBUG_BVR

	save_debug dbgbvr



	mov	x5, x25

	add	x4, x3, #DEBUG_WCR

	save_debug dbgwcr

	add	x4, x3, #DEBUG_WVR

	save_debug dbgwvr



	mrs	x21, mdccint_el1

	str	x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)]

	ret



/* Restore debug state */

__restore_debug:

	// x2: ptr to CPU context

	// x3: ptr to debug reg struct

	// x4/x5/x6-22/x24-26: trashed



	mrs	x26, id_aa64dfr0_el1

	ubfx	x24, x26, #12, #4	// Extract BRPs

	ubfx	x25, x26, #20, #4	// Extract WRPs

	mov	w26, #15

	sub	w24, w26, w24		// How many BPs to skip

	sub	w25, w26, w25		// How many WPs to skip



	mov	x5, x24

	add	x4, x3, #DEBUG_BCR

	restore_debug dbgbcr

	add	x4, x3, #DEBUG_BVR

	restore_debug dbgbvr



	mov	x5, x25

	add	x4, x3, #DEBUG_WCR

	restore_debug dbgwcr

	add	x4, x3, #DEBUG_WVR

	restore_debug dbgwvr



	ldr	x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)]

	msr	mdccint_el1, x21



	ret



__save_fpsimd:

	skip_fpsimd_state x3, 1f

	save_fpsimd

1:	ret



__restore_fpsimd:

	skip_fpsimd_state x3, 1f

	restore_fpsimd

1:	ret



switch_to_guest_fpsimd:

	push	x4, lr



	mrs	x2, cptr_el2

	bic	x2, x2, #CPTR_EL2_TFP

	msr	cptr_el2, x2

	isb



	mrs	x0, tpidr_el2



	ldr	x2, [x0, #VCPU_HOST_CONTEXT]

	kern_hyp_va x2

	bl __save_fpsimd



	add	x2, x0, #VCPU_CONTEXT

	bl __restore_fpsimd



	skip_32bit_state x3, 1f

	ldr	x4, [x2, #CPU_SYSREG_OFFSET(FPEXC32_EL2)]

	msr	fpexc32_el2, x4

1:

	pop	x4, lr

	pop	x2, x3

	pop	x0, x1



	eret



/*

 * u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu);

 *

 * This is the world switch. The first half of the function

 * deals with entering the guest, and anything from __kvm_vcpu_return

 * to the end of the function deals with reentering the host.

 * On the enter path, only x0 (vcpu pointer) must be preserved until

 * the last moment. On the exit path, x0 (vcpu pointer) and x1 (exception

 * code) must both be preserved until the epilogue.

 * In both cases, x2 points to the CPU context we're saving/restoring from/to.

 */

ENTRY(__kvm_vcpu_run)

	kern_hyp_va	x0

	msr	tpidr_el2, x0	// Save the vcpu register



	// Host context

	ldr	x2, [x0, #VCPU_HOST_CONTEXT]

	kern_hyp_va x2



	save_host_regs

	bl __save_sysregs



	compute_debug_state 1f

	add	x3, x0, #VCPU_HOST_DEBUG_STATE

	bl	__save_debug

1:

	activate_traps

	activate_vm



	restore_vgic_state

	restore_timer_state



	// Guest context

	add	x2, x0, #VCPU_CONTEXT



	// We must restore the 32-bit state before the sysregs, thanks

	// to Cortex-A57 erratum #852523.

	restore_guest_32bit_state

	bl __restore_sysregs



	skip_debug_state x3, 1f

	ldr	x3, [x0, #VCPU_DEBUG_PTR]

	kern_hyp_va x3

	bl	__restore_debug

1:

	restore_guest_regs



	// That's it, no more messing around.

	eret



__kvm_vcpu_return:

	// Assume x0 is the vcpu pointer, x1 the return code

	// Guest's x0-x3 are on the stack



	// Guest context

	add	x2, x0, #VCPU_CONTEXT



	save_guest_regs

	bl __save_fpsimd

	bl __save_sysregs



	skip_debug_state x3, 1f

	ldr	x3, [x0, #VCPU_DEBUG_PTR]

	kern_hyp_va x3

	bl	__save_debug

1:

	save_guest_32bit_state



	save_timer_state

	save_vgic_state



	deactivate_traps

	deactivate_vm



	// Host context

	ldr	x2, [x0, #VCPU_HOST_CONTEXT]

	kern_hyp_va x2



	bl __restore_sysregs

	bl __restore_fpsimd

	/* Clear FPSIMD and Trace trapping */

	msr     cptr_el2, xzr



	skip_debug_state x3, 1f

	// Clear the dirty flag for the next run, as all the state has

	// already been saved. Note that we nuke the whole 64bit word.

	// If we ever add more flags, we'll have to be more careful...

	str	xzr, [x0, #VCPU_DEBUG_FLAGS]

	add	x3, x0, #VCPU_HOST_DEBUG_STATE

	bl	__restore_debug

1:

	restore_host_regs



	mov	x0, x1

	ret

END(__kvm_vcpu_run)



// void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);

ENTRY(__kvm_tlb_flush_vmid_ipa)

	dsb	ishst



	kern_hyp_va	x0

	ldr	x2, [x0, #KVM_VTTBR]

	msr	vttbr_el2, x2

	isb



	/*

	 * We could do so much better if we had the VA as well.

	 * Instead, we invalidate Stage-2 for this IPA, and the

	 * whole of Stage-1. Weep...

	 */

	lsr	x1, x1, #12

	tlbi	ipas2e1is, x1

	/*

	 * We have to ensure completion of the invalidation at Stage-2,

	 * since a table walk on another CPU could refill a TLB with a

	 * complete (S1 + S2) walk based on the old Stage-2 mapping if

	 * the Stage-1 invalidation happened first.

	 */

	dsb	ish

	tlbi	vmalle1is

	dsb	ish

	isb



	msr	vttbr_el2, xzr

	ret

ENDPROC(__kvm_tlb_flush_vmid_ipa)



/**

 * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs

 * @struct kvm *kvm - pointer to kvm structure

 *

 * Invalidates all Stage 1 and 2 TLB entries for current VMID.

 */

ENTRY(__kvm_tlb_flush_vmid)

	dsb     ishst



	kern_hyp_va     x0

	ldr     x2, [x0, #KVM_VTTBR]

	msr     vttbr_el2, x2

	isb



	tlbi    vmalls12e1is

	dsb     ish

	isb



	msr     vttbr_el2, xzr

	ret

ENDPROC(__kvm_tlb_flush_vmid)



ENTRY(__kvm_flush_vm_context)

	dsb	ishst

	tlbi	alle1is

	ic	ialluis

	dsb	ish

	ret

ENDPROC(__kvm_flush_vm_context)



__kvm_hyp_panic:

	// Stash PAR_EL1 before corrupting it in __restore_sysregs

	mrs	x0, par_el1

	push	x0, xzr



	// Guess the context by looking at VTTBR:

	// If zero, then we're already a host.

	// Otherwise restore a minimal host context before panicing.

	mrs	x0, vttbr_el2

	cbz	x0, 1f



	mrs	x0, tpidr_el2



	deactivate_traps

	deactivate_vm



	ldr	x2, [x0, #VCPU_HOST_CONTEXT]

	kern_hyp_va x2



	bl __restore_sysregs



	/*

	 * Make sure we have a valid host stack, and don't leave junk in the

	 * frame pointer that will give us a misleading host stack unwinding.

	 */

	ldr	x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]

	msr	sp_el1, x22

	mov	x29, xzr



1:	adr	x0, __hyp_panic_str

	adr	x1, 2f

	ldp	x2, x3, [x1]

	sub	x0, x0, x2

	add	x0, x0, x3

	mrs	x1, spsr_el2

	mrs	x2, elr_el2

	mrs	x3, esr_el2

	mrs	x4, far_el2

	mrs	x5, hpfar_el2

	pop	x6, xzr		// active context PAR_EL1

	mrs	x7, tpidr_el2



	mov	lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\

		      PSR_MODE_EL1h)

	msr	spsr_el2, lr

	ldr	lr, =panic

	msr	elr_el2, lr

	eret



	.align	3

2:	.quad	HYP_PAGE_OFFSET

	.quad	PAGE_OFFSET

ENDPROC(__kvm_hyp_panic)



__hyp_panic_str:

	.ascii	"HYP panic:\nPS:%08x PC:%016x ESR:%08x\nFAR:%016x HPFAR:%016x PAR:%016x\nVCPU:%p\n\0"



	.align	2



/*

 * u64 kvm_call_hyp(void *hypfn, ...);
@@ -934,7 +31,7 @@ __hyp_panic_str: 
 * passed as x0, x1, and x2 (a maximum of 3 arguments in addition to the

 * function pointer can be passed).  The function being called must be mapped

 * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c).  Return values are

 * passed in r0 and r1.

 * passed in x0.

 *

 * A function pointer with a value of 0 has a special meaning, and is

 * used to implement __hyp_get_vectors in the same way as in
@@ -944,179 +41,3 @@ ENTRY(kvm_call_hyp) 
	hvc	#0

	ret

ENDPROC(kvm_call_hyp)
 


.macro invalid_vector	label, target

	.align	2

\label:

	b \target

ENDPROC(\label)

.endm



	/* None of these should ever happen */

	invalid_vector	el2t_sync_invalid, __kvm_hyp_panic

	invalid_vector	el2t_irq_invalid, __kvm_hyp_panic

	invalid_vector	el2t_fiq_invalid, __kvm_hyp_panic

	invalid_vector	el2t_error_invalid, __kvm_hyp_panic

	invalid_vector	el2h_sync_invalid, __kvm_hyp_panic

	invalid_vector	el2h_irq_invalid, __kvm_hyp_panic

	invalid_vector	el2h_fiq_invalid, __kvm_hyp_panic

	invalid_vector	el2h_error_invalid, __kvm_hyp_panic

	invalid_vector	el1_sync_invalid, __kvm_hyp_panic

	invalid_vector	el1_irq_invalid, __kvm_hyp_panic

	invalid_vector	el1_fiq_invalid, __kvm_hyp_panic

	invalid_vector	el1_error_invalid, __kvm_hyp_panic



el1_sync:					// Guest trapped into EL2

	push	x0, x1

	push	x2, x3



	mrs	x1, esr_el2

	lsr	x2, x1, #ESR_ELx_EC_SHIFT



	cmp	x2, #ESR_ELx_EC_HVC64

	b.ne	el1_trap



	mrs	x3, vttbr_el2			// If vttbr is valid, the 64bit guest

	cbnz	x3, el1_trap			// called HVC



	/* Here, we're pretty sure the host called HVC. */

	pop	x2, x3

	pop	x0, x1



	/* Check for __hyp_get_vectors */

	cbnz	x0, 1f

	mrs	x0, vbar_el2

	b	2f



1:	push	lr, xzr



	/*

	 * Compute the function address in EL2, and shuffle the parameters.

	 */

	kern_hyp_va	x0

	mov	lr, x0

	mov	x0, x1

	mov	x1, x2

	mov	x2, x3

	blr	lr



	pop	lr, xzr

2:	eret



el1_trap:

	/*

	 * x1: ESR

	 * x2: ESR_EC

	 */



	/* Guest accessed VFP/SIMD registers, save host, restore Guest */

	cmp	x2, #ESR_ELx_EC_FP_ASIMD

	b.eq	switch_to_guest_fpsimd



	cmp	x2, #ESR_ELx_EC_DABT_LOW

	mov	x0, #ESR_ELx_EC_IABT_LOW

	ccmp	x2, x0, #4, ne

	b.ne	1f		// Not an abort we care about



	/* This is an abort. Check for permission fault */

alternative_if_not ARM64_WORKAROUND_834220

	and	x2, x1, #ESR_ELx_FSC_TYPE

	cmp	x2, #FSC_PERM

	b.ne	1f		// Not a permission fault

alternative_else

	nop			// Use the permission fault path to

	nop			// check for a valid S1 translation,

	nop			// regardless of the ESR value.

alternative_endif



	/*

	 * Check for Stage-1 page table walk, which is guaranteed

	 * to give a valid HPFAR_EL2.

	 */

	tbnz	x1, #7, 1f	// S1PTW is set



	/* Preserve PAR_EL1 */

	mrs	x3, par_el1

	push	x3, xzr



	/*

	 * Permission fault, HPFAR_EL2 is invalid.

	 * Resolve the IPA the hard way using the guest VA.

	 * Stage-1 translation already validated the memory access rights.

	 * As such, we can use the EL1 translation regime, and don't have

	 * to distinguish between EL0 and EL1 access.

	 */

	mrs	x2, far_el2

	at	s1e1r, x2

	isb



	/* Read result */

	mrs	x3, par_el1

	pop	x0, xzr			// Restore PAR_EL1 from the stack

	msr	par_el1, x0

	tbnz	x3, #0, 3f		// Bail out if we failed the translation

	ubfx	x3, x3, #12, #36	// Extract IPA

	lsl	x3, x3, #4		// and present it like HPFAR

	b	2f



1:	mrs	x3, hpfar_el2

	mrs	x2, far_el2



2:	mrs	x0, tpidr_el2

	str	w1, [x0, #VCPU_ESR_EL2]

	str	x2, [x0, #VCPU_FAR_EL2]

	str	x3, [x0, #VCPU_HPFAR_EL2]



	mov	x1, #ARM_EXCEPTION_TRAP

	b	__kvm_vcpu_return



	/*

	 * Translation failed. Just return to the guest and

	 * let it fault again. Another CPU is probably playing

	 * behind our back.

	 */

3:	pop	x2, x3

	pop	x0, x1



	eret



el1_irq:

	push	x0, x1

	push	x2, x3

	mrs	x0, tpidr_el2

	mov	x1, #ARM_EXCEPTION_IRQ

	b	__kvm_vcpu_return



	.ltorg



	.align 11



ENTRY(__kvm_hyp_vector)

	ventry	el2t_sync_invalid		// Synchronous EL2t

	ventry	el2t_irq_invalid		// IRQ EL2t

	ventry	el2t_fiq_invalid		// FIQ EL2t

	ventry	el2t_error_invalid		// Error EL2t



	ventry	el2h_sync_invalid		// Synchronous EL2h

	ventry	el2h_irq_invalid		// IRQ EL2h

	ventry	el2h_fiq_invalid		// FIQ EL2h

	ventry	el2h_error_invalid		// Error EL2h



	ventry	el1_sync			// Synchronous 64-bit EL1

	ventry	el1_irq				// IRQ 64-bit EL1

	ventry	el1_fiq_invalid			// FIQ 64-bit EL1

	ventry	el1_error_invalid		// Error 64-bit EL1



	ventry	el1_sync			// Synchronous 32-bit EL1

	ventry	el1_irq				// IRQ 32-bit EL1

	ventry	el1_fiq_invalid			// FIQ 32-bit EL1

	ventry	el1_error_invalid		// Error 32-bit EL1

ENDPROC(__kvm_hyp_vector)





ENTRY(__kvm_get_mdcr_el2)

	mrs	x0, mdcr_el2

	ret

ENDPROC(__kvm_get_mdcr_el2)



	.popsection