ARM: KVM: Remove the old world switch

 */

#include <linux/linkage.h>

#include <linux/const.h>

#include <asm/unified.h>

#include <asm/page.h>

#include <asm/ptrace.h>

#include <asm/asm-offsets.h>

#include <asm/kvm_asm.h>

#include <asm/kvm_arm.h>

#include <asm/vfpmacros.h>

#include "interrupts_head.S"

	.text

	.pushsection	.hyp.text, "ax"

/********************************************************************

 * Flush per-VMID TLBs

 *

 * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);

 *

 * We rely on the hardware to broadcast the TLB invalidation to all CPUs

 * inside the inner-shareable domain (which is the case for all v7

 * implementations).  If we come across a non-IS SMP implementation, we'll

 * have to use an IPI based mechanism. Until then, we stick to the simple

 * hardware assisted version.

 *

 * As v7 does not support flushing per IPA, just nuke the whole TLB

 * instead, ignoring the ipa value.

 */

ENTRY(__kvm_tlb_flush_vmid_ipa)

	push	{r2, r3}

	dsb	ishst

	add	r0, r0, #KVM_VTTBR

	ldrd	r2, r3, [r0]

	mcrr	p15, 6, rr_lo_hi(r2, r3), c2	@ Write VTTBR

	isb

	mcr     p15, 0, r0, c8, c3, 0	@ TLBIALLIS (rt ignored)

	dsb	ish

	isb

	mov	r2, #0

	mov	r3, #0

	mcrr	p15, 6, r2, r3, c2	@ Back to VMID #0

	isb				@ Not necessary if followed by eret

	pop	{r2, r3}

	bx	lr

ENDPROC(__kvm_tlb_flush_vmid_ipa)

/**

 * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs

 *

 * Reuses __kvm_tlb_flush_vmid_ipa() for ARMv7, without passing address

 * parameter

 */

ENTRY(__kvm_tlb_flush_vmid)

	b	__kvm_tlb_flush_vmid_ipa

ENDPROC(__kvm_tlb_flush_vmid)

/********************************************************************

 * Flush TLBs and instruction caches of all CPUs inside the inner-shareable

 * domain, for all VMIDs

 *

 * void __kvm_flush_vm_context(void);

 */

ENTRY(__kvm_flush_vm_context)

	mov	r0, #0			@ rn parameter for c15 flushes is SBZ

	/* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */

	mcr     p15, 4, r0, c8, c3, 4

	/* Invalidate instruction caches Inner Shareable (ICIALLUIS) */

	mcr     p15, 0, r0, c7, c1, 0

	dsb	ish

	isb				@ Not necessary if followed by eret

	bx	lr

ENDPROC(__kvm_flush_vm_context)

/********************************************************************

 *  Hypervisor world-switch code

 *

 *

 * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)

 */

ENTRY(__kvm_vcpu_run)

	@ Save the vcpu pointer

	mcr	p15, 4, vcpu, c13, c0, 2	@ HTPIDR

	save_host_regs

	restore_vgic_state

	restore_timer_state

	@ Store hardware CP15 state and load guest state

	read_cp15_state store_to_vcpu = 0

	write_cp15_state read_from_vcpu = 1

	@ If the host kernel has not been configured with VFPv3 support,

	@ then it is safer if we deny guests from using it as well.

#ifdef CONFIG_VFPv3

	@ Set FPEXC_EN so the guest doesn't trap floating point instructions

	VFPFMRX r2, FPEXC		@ VMRS

	push	{r2}

	orr	r2, r2, #FPEXC_EN

	VFPFMXR FPEXC, r2		@ VMSR

#endif

	@ Configure Hyp-role

	configure_hyp_role vmentry

	@ Trap coprocessor CRx accesses

	set_hstr vmentry

	set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))

	set_hdcr vmentry

	@ Write configured ID register into MIDR alias

	ldr	r1, [vcpu, #VCPU_MIDR]

	mcr	p15, 4, r1, c0, c0, 0

	@ Write guest view of MPIDR into VMPIDR

	ldr	r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]

	mcr	p15, 4, r1, c0, c0, 5

	@ Set up guest memory translation

	ldr	r1, [vcpu, #VCPU_KVM]

	add	r1, r1, #KVM_VTTBR

	ldrd	r2, r3, [r1]

	mcrr	p15, 6, rr_lo_hi(r2, r3), c2	@ Write VTTBR

	@ We're all done, just restore the GPRs and go to the guest

	restore_guest_regs

	clrex				@ Clear exclusive monitor

	eret

__kvm_vcpu_return:

	/*

	 * return convention:

	 * guest r0, r1, r2 saved on the stack

	 * r0: vcpu pointer

	 * r1: exception code

	 */

	save_guest_regs

	@ Set VMID == 0

	mov	r2, #0

	mov	r3, #0

	mcrr	p15, 6, r2, r3, c2	@ Write VTTBR

	@ Don't trap coprocessor accesses for host kernel

	set_hstr vmexit

	set_hdcr vmexit

	set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11)), after_vfp_restore

#ifdef CONFIG_VFPv3

	@ Switch VFP/NEON hardware state to the host's

	add	r7, vcpu, #(VCPU_GUEST_CTXT + CPU_CTXT_VFP)

	store_vfp_state r7

	add	r7, vcpu, #VCPU_HOST_CTXT

	ldr	r7, [r7]

	add	r7, r7, #CPU_CTXT_VFP

	restore_vfp_state r7

after_vfp_restore:

	@ Restore FPEXC_EN which we clobbered on entry

	pop	{r2}

	VFPFMXR FPEXC, r2

#else

after_vfp_restore:

#endif

	@ Reset Hyp-role

	configure_hyp_role vmexit

	@ Let host read hardware MIDR

	mrc	p15, 0, r2, c0, c0, 0

	mcr	p15, 4, r2, c0, c0, 0

	@ Back to hardware MPIDR

	mrc	p15, 0, r2, c0, c0, 5

	mcr	p15, 4, r2, c0, c0, 5

	@ Store guest CP15 state and restore host state

	read_cp15_state store_to_vcpu = 1

	write_cp15_state read_from_vcpu = 0

	save_timer_state

	save_vgic_state

	restore_host_regs

	clrex				@ Clear exclusive monitor

	mov	r0, r1			@ Return the return code

	bx	lr			@ return to IOCTL

/********************************************************************

 *  Call function in Hyp mode

ENTRY(kvm_call_hyp)

	hvc	#0

	bx	lr

/********************************************************************

 * Hypervisor exception vector and handlers

 *

 *

 * The KVM/ARM Hypervisor ABI is defined as follows:

 *

 * Entry to Hyp mode from the host kernel will happen _only_ when an HVC

 * instruction is issued since all traps are disabled when running the host

 * kernel as per the Hyp-mode initialization at boot time.

 *

 * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc

 * below) when the HVC instruction is called from SVC mode (i.e. a guest or the

 * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC

 * instructions are called from within Hyp-mode.

 *

 * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):

 *    Switching to Hyp mode is done through a simple HVC #0 instruction. The

 *    exception vector code will check that the HVC comes from VMID==0 and if

 *    so will push the necessary state (SPSR, lr_usr) on the Hyp stack.

 *    - r0 contains a pointer to a HYP function

 *    - r1, r2, and r3 contain arguments to the above function.

 *    - The HYP function will be called with its arguments in r0, r1 and r2.

 *    On HYP function return, we return directly to SVC.

 *

 * Note that the above is used to execute code in Hyp-mode from a host-kernel

 * point of view, and is a different concept from performing a world-switch and

 * executing guest code SVC mode (with a VMID != 0).

 */

/* Handle undef, svc, pabt, or dabt by crashing with a user notice */

.macro bad_exception exception_code, panic_str

	push	{r0-r2}

	mrrc	p15, 6, r0, r1, c2	@ Read VTTBR

	lsr	r1, r1, #16

	ands	r1, r1, #0xff

	beq	99f

	load_vcpu			@ Load VCPU pointer

	.if \exception_code == ARM_EXCEPTION_DATA_ABORT

	mrc	p15, 4, r2, c5, c2, 0	@ HSR

	mrc	p15, 4, r1, c6, c0, 0	@ HDFAR

	str	r2, [vcpu, #VCPU_HSR]

	str	r1, [vcpu, #VCPU_HxFAR]

	.endif

	.if \exception_code == ARM_EXCEPTION_PREF_ABORT

	mrc	p15, 4, r2, c5, c2, 0	@ HSR

	mrc	p15, 4, r1, c6, c0, 2	@ HIFAR

	str	r2, [vcpu, #VCPU_HSR]

	str	r1, [vcpu, #VCPU_HxFAR]

	.endif

	mov	r1, #\exception_code

	b	__kvm_vcpu_return

	@ We were in the host already. Let's craft a panic-ing return to SVC.

99:	mrs	r2, cpsr

	bic	r2, r2, #MODE_MASK

	orr	r2, r2, #SVC_MODE

THUMB(	orr	r2, r2, #PSR_T_BIT	)

	msr	spsr_cxsf, r2

	mrs	r1, ELR_hyp

	ldr	r2, =panic

	msr	ELR_hyp, r2

	ldr	r0, =\panic_str

	clrex				@ Clear exclusive monitor

	eret

.endm

	.align 5

__kvm_hyp_vector:

	.globl __kvm_hyp_vector

	@ Hyp-mode exception vector

	W(b)	hyp_reset

	W(b)	hyp_undef

	W(b)	hyp_svc

	W(b)	hyp_pabt

	W(b)	hyp_dabt

	W(b)	hyp_hvc

	W(b)	hyp_irq

	W(b)	hyp_fiq

	.align

hyp_reset:

	b	hyp_reset

	.align

hyp_undef:

	bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str

	.align

hyp_svc:

	bad_exception ARM_EXCEPTION_HVC, svc_die_str

	.align

hyp_pabt:

	bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str

	.align

hyp_dabt:

	bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str

	.align

hyp_hvc:

	/*

	 * Getting here is either becuase of a trap from a guest or from calling

	 * HVC from the host kernel, which means "switch to Hyp mode".

	 */

	push	{r0, r1, r2}

	@ Check syndrome register

	mrc	p15, 4, r1, c5, c2, 0	@ HSR

	lsr	r0, r1, #HSR_EC_SHIFT

	cmp	r0, #HSR_EC_HVC

	bne	guest_trap		@ Not HVC instr.

	/*

	 * Let's check if the HVC came from VMID 0 and allow simple

	 * switch to Hyp mode

	 */

	mrrc    p15, 6, r0, r2, c2

	lsr     r2, r2, #16

	and     r2, r2, #0xff

	cmp     r2, #0

	bne	guest_trap		@ Guest called HVC

	/*

	 * Getting here means host called HVC, we shift parameters and branch

	 * to Hyp function.

	 */

	pop	{r0, r1, r2}

	/* Check for __hyp_get_vectors */

	cmp	r0, #-1

	mrceq	p15, 4, r0, c12, c0, 0	@ get HVBAR

	beq	1f

	push	{lr}

	mrs	lr, SPSR

	push	{lr}

	mov	lr, r0

	mov	r0, r1

	mov	r1, r2

	mov	r2, r3

THUMB(	orr	lr, #1)

	blx	lr			@ Call the HYP function

	pop	{lr}

	msr	SPSR_csxf, lr

	pop	{lr}

1:	eret

guest_trap:

	load_vcpu			@ Load VCPU pointer to r0

	str	r1, [vcpu, #VCPU_HSR]

	@ Check if we need the fault information

	lsr	r1, r1, #HSR_EC_SHIFT

#ifdef CONFIG_VFPv3

	cmp	r1, #HSR_EC_CP_0_13

	beq	switch_to_guest_vfp

#endif

	cmp	r1, #HSR_EC_IABT

	mrceq	p15, 4, r2, c6, c0, 2	@ HIFAR

	beq	2f

	cmp	r1, #HSR_EC_DABT

	bne	1f

	mrc	p15, 4, r2, c6, c0, 0	@ HDFAR

2:	str	r2, [vcpu, #VCPU_HxFAR]

	/*

	 * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:

	 *

	 * Abort on the stage 2 translation for a memory access from a

	 * Non-secure PL1 or PL0 mode:

	 *

	 * For any Access flag fault or Translation fault, and also for any

	 * Permission fault on the stage 2 translation of a memory access

	 * made as part of a translation table walk for a stage 1 translation,

	 * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR

	 * is UNKNOWN.

	 */

	/* Check for permission fault, and S1PTW */

	mrc	p15, 4, r1, c5, c2, 0	@ HSR

	and	r0, r1, #HSR_FSC_TYPE

	cmp	r0, #FSC_PERM

	tsteq	r1, #(1 << 7)		@ S1PTW

	mrcne	p15, 4, r2, c6, c0, 4	@ HPFAR

	bne	3f

	/* Preserve PAR */

	mrrc	p15, 0, r0, r1, c7	@ PAR

	push	{r0, r1}

	/* Resolve IPA using the xFAR */

	mcr	p15, 0, r2, c7, c8, 0	@ ATS1CPR

	isb

	mrrc	p15, 0, r0, r1, c7	@ PAR

	tst	r0, #1

	bne	4f			@ Failed translation

	ubfx	r2, r0, #12, #20

	lsl	r2, r2, #4

	orr	r2, r2, r1, lsl #24

	/* Restore PAR */

	pop	{r0, r1}

	mcrr	p15, 0, r0, r1, c7	@ PAR

3:	load_vcpu			@ Load VCPU pointer to r0

	str	r2, [r0, #VCPU_HPFAR]

1:	mov	r1, #ARM_EXCEPTION_HVC

	b	__kvm_vcpu_return

4:	pop	{r0, r1}		@ Failed translation, return to guest

	mcrr	p15, 0, r0, r1, c7	@ PAR

	clrex

	pop	{r0, r1, r2}

	eret

/*

 * If VFPv3 support is not available, then we will not switch the VFP

 * registers; however cp10 and cp11 accesses will still trap and fallback

 * to the regular coprocessor emulation code, which currently will

 * inject an undefined exception to the guest.

 */

#ifdef CONFIG_VFPv3

switch_to_guest_vfp:

	push	{r3-r7}

	@ NEON/VFP used.  Turn on VFP access.

	set_hcptr vmtrap, (HCPTR_TCP(10) | HCPTR_TCP(11))

	@ Switch VFP/NEON hardware state to the guest's

	add	r7, r0, #VCPU_HOST_CTXT

	ldr	r7, [r7]

	add	r7, r7, #CPU_CTXT_VFP

	store_vfp_state r7

	add	r7, r0, #(VCPU_GUEST_CTXT + CPU_CTXT_VFP)

	restore_vfp_state r7

	pop	{r3-r7}

	pop	{r0-r2}

	clrex

	eret

#endif

	.align

hyp_irq:

	push	{r0, r1, r2}

	mov	r1, #ARM_EXCEPTION_IRQ

	load_vcpu			@ Load VCPU pointer to r0

	b	__kvm_vcpu_return

	.align

hyp_fiq:

	b	hyp_fiq

	.ltorg

	.popsection

	.pushsection ".rodata"

und_die_str:

	.ascii	"unexpected undefined exception in Hyp mode at: %#08x\n"

pabt_die_str:

	.ascii	"unexpected prefetch abort in Hyp mode at: %#08x\n"

dabt_die_str:

	.ascii	"unexpected data abort in Hyp mode at: %#08x\n"

svc_die_str:

	.ascii	"unexpected HVC/SVC trap in Hyp mode at: %#08x\n"

	.popsection

ENDPROC(kvm_call_hyp)