Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c7c5cbb4 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman
Browse files

powerpc/64s/exception: move interrupt entry code above the common handler 



This better reflects the order in which the code is executed.

No generated code change except BUG line number constants.

Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20190802105709.27696-34-npiggin@gmail.com
parent d1a84718

arch/powerpc/kernel/exceptions-64s.S

+225 −225
Original line number Diff line number Diff line
@@ -180,6 +180,130 @@ BEGIN_FTR_SECTION_NESTED(943) \ 
	std	ra,offset(r13);						\

END_FTR_SECTION_NESTED(ftr,ftr,943)



/*

 * Branch to label using its 0xC000 address. This results in instruction

 * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned

 * on using mtmsr rather than rfid.

 *

 * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than

 * load KBASE for a slight optimisation.

 */

#define BRANCH_TO_C000(reg, label)					\

	__LOAD_FAR_HANDLER(reg, label);					\

	mtctr	reg;							\

	bctr



.macro INT_KVM_HANDLER vec, hsrr, area, skip

	.if \hsrr

	TRAMP_KVM_BEGIN(do_kvm_H\vec\())

	.else

	TRAMP_KVM_BEGIN(do_kvm_\vec\())

	.endif

	KVM_HANDLER \vec, \hsrr, \area, \skip

.endm



#ifdef CONFIG_KVM_BOOK3S_64_HANDLER

#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE

/*

 * If hv is possible, interrupts come into to the hv version

 * of the kvmppc_interrupt code, which then jumps to the PR handler,

 * kvmppc_interrupt_pr, if the guest is a PR guest.

 */

#define kvmppc_interrupt kvmppc_interrupt_hv

#else

#define kvmppc_interrupt kvmppc_interrupt_pr

#endif



.macro KVMTEST hsrr, n

	lbz	r10,HSTATE_IN_GUEST(r13)

	cmpwi	r10,0

	.if \hsrr == EXC_HV_OR_STD

	BEGIN_FTR_SECTION

	bne	do_kvm_H\n

	FTR_SECTION_ELSE

	bne	do_kvm_\n

	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)

	.elseif \hsrr

	bne	do_kvm_H\n

	.else

	bne	do_kvm_\n

	.endif

.endm



.macro KVM_HANDLER vec, hsrr, area, skip

	.if \skip

	cmpwi	r10,KVM_GUEST_MODE_SKIP

	beq	89f

	.else

BEGIN_FTR_SECTION_NESTED(947)

	ld	r10,\area+EX_CFAR(r13)

	std	r10,HSTATE_CFAR(r13)

END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947)

	.endif



BEGIN_FTR_SECTION_NESTED(948)

	ld	r10,\area+EX_PPR(r13)

	std	r10,HSTATE_PPR(r13)

END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)

	ld	r10,\area+EX_R10(r13)

	std	r12,HSTATE_SCRATCH0(r13)

	sldi	r12,r9,32

	/* HSRR variants have the 0x2 bit added to their trap number */

	.if \hsrr == EXC_HV_OR_STD

	BEGIN_FTR_SECTION

	ori	r12,r12,(\vec + 0x2)

	FTR_SECTION_ELSE

	ori	r12,r12,(\vec)

	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)

	.elseif \hsrr

	ori	r12,r12,(\vec + 0x2)

	.else

	ori	r12,r12,(\vec)

	.endif



#ifdef CONFIG_RELOCATABLE

	/*

	 * KVM requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives

	 * outside the head section. CONFIG_RELOCATABLE KVM expects CTR

	 * to be saved in HSTATE_SCRATCH1.

	 */

	mfctr	r9

	std	r9,HSTATE_SCRATCH1(r13)

	__LOAD_FAR_HANDLER(r9, kvmppc_interrupt)

	mtctr	r9

	ld	r9,\area+EX_R9(r13)

	bctr

#else

	ld	r9,\area+EX_R9(r13)

	b	kvmppc_interrupt

#endif





	.if \skip

89:	mtocrf	0x80,r9

	ld	r9,\area+EX_R9(r13)

	ld	r10,\area+EX_R10(r13)

	.if \hsrr == EXC_HV_OR_STD

	BEGIN_FTR_SECTION

	b	kvmppc_skip_Hinterrupt

	FTR_SECTION_ELSE

	b	kvmppc_skip_interrupt

	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)

	.elseif \hsrr

	b	kvmppc_skip_Hinterrupt

	.else

	b	kvmppc_skip_interrupt

	.endif

	.endif

.endm



#else

.macro KVMTEST hsrr, n

.endm

.macro KVM_HANDLER vec, hsrr, area, skip

.endm

#endif



.macro INT_SAVE_SRR_AND_JUMP label, hsrr, set_ri

	ld	r10,PACAKMSR(r13)	/* get MSR value for kernel */

	.if ! \set_ri
@@ -276,119 +400,125 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) 
.endm



/*

 * Branch to label using its 0xC000 address. This results in instruction

 * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned

 * on using mtmsr rather than rfid.

 * This is the BOOK3S interrupt entry code macro.

 *

 * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than

 * load KBASE for a slight optimisation.

 */

#define BRANCH_TO_C000(reg, label)					\

	__LOAD_FAR_HANDLER(reg, label);					\

	mtctr	reg;							\

	bctr



#ifdef CONFIG_KVM_BOOK3S_64_HANDLER

#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE

/*

 * If hv is possible, interrupts come into to the hv version

 * of the kvmppc_interrupt code, which then jumps to the PR handler,

 * kvmppc_interrupt_pr, if the guest is a PR guest.

 * This can result in one of several things happening:

 * - Branch to the _common handler, relocated, in virtual mode.

 *   These are normal interrupts (synchronous and asynchronous) handled by

 *   the kernel.

 * - Branch to KVM, relocated but real mode interrupts remain in real mode.

 *   These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by

 *   / intended for host or guest kernel, but KVM must always be involved

 *   because the machine state is set for guest execution.

 * - Branch to the masked handler, unrelocated.

 *   These occur when maskable asynchronous interrupts are taken with the

 *   irq_soft_mask set.

 * - Branch to an "early" handler in real mode but relocated.

 *   This is done if early=1. MCE and HMI use these to handle errors in real

 *   mode.

 * - Fall through and continue executing in real, unrelocated mode.

 *   This is done if early=2.

 */

#define kvmppc_interrupt kvmppc_interrupt_hv

#else

#define kvmppc_interrupt kvmppc_interrupt_pr

#endif



.macro KVMTEST hsrr, n

	lbz	r10,HSTATE_IN_GUEST(r13)

	cmpwi	r10,0

	.if \hsrr == EXC_HV_OR_STD

	BEGIN_FTR_SECTION

	bne	do_kvm_H\n

	FTR_SECTION_ELSE

	bne	do_kvm_\n

	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)

	.elseif \hsrr

	bne	do_kvm_H\n

.macro INT_HANDLER name, vec, ool=0, early=0, virt=0, hsrr=0, area=PACA_EXGEN, ri=1, dar=0, dsisr=0, bitmask=0, kvm=0

	SET_SCRATCH0(r13)			/* save r13 */

	GET_PACA(r13)

	std	r9,\area\()+EX_R9(r13)		/* save r9 */

	OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR)

	HMT_MEDIUM

	std	r10,\area\()+EX_R10(r13)	/* save r10 - r12 */

	OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR)

	.if \ool

	.if !\virt

	b	tramp_real_\name

	.pushsection .text

	TRAMP_REAL_BEGIN(tramp_real_\name)

	.else

	bne	do_kvm_\n

	b	tramp_virt_\name

	.pushsection .text

	TRAMP_VIRT_BEGIN(tramp_virt_\name)

	.endif

	.endif

.endm



.macro KVM_HANDLER vec, hsrr, area, skip

	.if \skip

	cmpwi	r10,KVM_GUEST_MODE_SKIP

	beq	89f

	OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR)

	OPT_SAVE_REG_TO_PACA(\area\()+EX_CFAR, r10, CPU_FTR_CFAR)

	INTERRUPT_TO_KERNEL

	SAVE_CTR(r10, \area\())

	mfcr	r9

	.if \kvm

		KVMTEST \hsrr \vec

	.endif

	.if \bitmask

		lbz	r10,PACAIRQSOFTMASK(r13)

		andi.	r10,r10,\bitmask

		/* Associate vector numbers with bits in paca->irq_happened */

		.if \vec == 0x500 || \vec == 0xea0

		li	r10,PACA_IRQ_EE

		.elseif \vec == 0x900

		li	r10,PACA_IRQ_DEC

		.elseif \vec == 0xa00 || \vec == 0xe80

		li	r10,PACA_IRQ_DBELL

		.elseif \vec == 0xe60

		li	r10,PACA_IRQ_HMI

		.elseif \vec == 0xf00

		li	r10,PACA_IRQ_PMI

		.else

BEGIN_FTR_SECTION_NESTED(947)

	ld	r10,\area+EX_CFAR(r13)

	std	r10,HSTATE_CFAR(r13)

END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947)

		.abort "Bad maskable vector"

		.endif



BEGIN_FTR_SECTION_NESTED(948)

	ld	r10,\area+EX_PPR(r13)

	std	r10,HSTATE_PPR(r13)

END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)

	ld	r10,\area+EX_R10(r13)

	std	r12,HSTATE_SCRATCH0(r13)

	sldi	r12,r9,32

	/* HSRR variants have the 0x2 bit added to their trap number */

		.if \hsrr == EXC_HV_OR_STD

		BEGIN_FTR_SECTION

	ori	r12,r12,(\vec + 0x2)

		bne	masked_Hinterrupt

		FTR_SECTION_ELSE

	ori	r12,r12,(\vec)

		bne	masked_interrupt

		ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)

		.elseif \hsrr

	ori	r12,r12,(\vec + 0x2)

		bne	masked_Hinterrupt

		.else

	ori	r12,r12,(\vec)

		bne	masked_interrupt

		.endif

	.endif



#ifdef CONFIG_RELOCATABLE

	std	r11,\area\()+EX_R11(r13)

	std	r12,\area\()+EX_R12(r13)



	/*

	 * KVM requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives

	 * outside the head section. CONFIG_RELOCATABLE KVM expects CTR

	 * to be saved in HSTATE_SCRATCH1.

	 * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI],

	 * because a d-side MCE will clobber those registers so is

	 * not recoverable if they are live.

	 */

	mfctr	r9

	std	r9,HSTATE_SCRATCH1(r13)

	__LOAD_FAR_HANDLER(r9, kvmppc_interrupt)

	mtctr	r9

	ld	r9,\area+EX_R9(r13)

	bctr

#else

	ld	r9,\area+EX_R9(r13)

	b	kvmppc_interrupt

#endif





	.if \skip

89:	mtocrf	0x80,r9

	ld	r9,\area+EX_R9(r13)

	ld	r10,\area+EX_R10(r13)

	.if \hsrr == EXC_HV_OR_STD

	BEGIN_FTR_SECTION

	b	kvmppc_skip_Hinterrupt

	FTR_SECTION_ELSE

	b	kvmppc_skip_interrupt

	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)

	.elseif \hsrr

	b	kvmppc_skip_Hinterrupt

	GET_SCRATCH0(r10)

	std	r10,\area\()+EX_R13(r13)

	.if \dar

	.if \hsrr

	mfspr	r10,SPRN_HDAR

	.else

	b	kvmppc_skip_interrupt

	mfspr	r10,SPRN_DAR

	.endif

	std	r10,\area\()+EX_DAR(r13)

	.endif

	.if \dsisr

	.if \hsrr

	mfspr	r10,SPRN_HDSISR

	.else

	mfspr	r10,SPRN_DSISR

	.endif

	stw	r10,\area\()+EX_DSISR(r13)

	.endif

.endm



#else

.macro KVMTEST hsrr, n

.endm

.macro KVM_HANDLER vec, hsrr, area, skip

	.if \early == 2

	/* nothing more */

	.elseif \early

	mfctr	r10			/* save ctr, even for !RELOCATABLE */

	BRANCH_TO_C000(r11, \name\()_early_common)

	.elseif !\virt

	INT_SAVE_SRR_AND_JUMP \name\()_common, \hsrr, \ri

	.else

	INT_VIRT_SAVE_SRR_AND_JUMP \name\()_common, \hsrr

	.endif

	.if \ool

	.popsection

	.endif

.endm

#endif



/*

 * On entry r13 points to the paca, r9-r13 are saved in the paca,
@@ -555,136 +685,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) 
#define FINISH_NAP

#endif



/*

 * This is the BOOK3S interrupt entry code macro.

 *

 * This can result in one of several things happening:

 * - Branch to the _common handler, relocated, in virtual mode.

 *   These are normal interrupts (synchronous and asynchronous) handled by

 *   the kernel.

 * - Branch to KVM, relocated but real mode interrupts remain in real mode.

 *   These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by

 *   / intended for host or guest kernel, but KVM must always be involved

 *   because the machine state is set for guest execution.

 * - Branch to the masked handler, unrelocated.

 *   These occur when maskable asynchronous interrupts are taken with the

 *   irq_soft_mask set.

 * - Branch to an "early" handler in real mode but relocated.

 *   This is done if early=1. MCE and HMI use these to handle errors in real

 *   mode.

 * - Fall through and continue executing in real, unrelocated mode.

 *   This is done if early=2.

 */

.macro INT_HANDLER name, vec, ool=0, early=0, virt=0, hsrr=0, area=PACA_EXGEN, ri=1, dar=0, dsisr=0, bitmask=0, kvm=0

	SET_SCRATCH0(r13)			/* save r13 */

	GET_PACA(r13)

	std	r9,\area\()+EX_R9(r13)		/* save r9 */

	OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR)

	HMT_MEDIUM

	std	r10,\area\()+EX_R10(r13)	/* save r10 - r12 */

	OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR)

	.if \ool

	.if !\virt

	b	tramp_real_\name

	.pushsection .text

	TRAMP_REAL_BEGIN(tramp_real_\name)

	.else

	b	tramp_virt_\name

	.pushsection .text

	TRAMP_VIRT_BEGIN(tramp_virt_\name)

	.endif

	.endif



	OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR)

	OPT_SAVE_REG_TO_PACA(\area\()+EX_CFAR, r10, CPU_FTR_CFAR)

	INTERRUPT_TO_KERNEL

	SAVE_CTR(r10, \area\())

	mfcr	r9

	.if \kvm

		KVMTEST \hsrr \vec

	.endif

	.if \bitmask

		lbz	r10,PACAIRQSOFTMASK(r13)

		andi.	r10,r10,\bitmask

		/* Associate vector numbers with bits in paca->irq_happened */

		.if \vec == 0x500 || \vec == 0xea0

		li	r10,PACA_IRQ_EE

		.elseif \vec == 0x900

		li	r10,PACA_IRQ_DEC

		.elseif \vec == 0xa00 || \vec == 0xe80

		li	r10,PACA_IRQ_DBELL

		.elseif \vec == 0xe60

		li	r10,PACA_IRQ_HMI

		.elseif \vec == 0xf00

		li	r10,PACA_IRQ_PMI

		.else

		.abort "Bad maskable vector"

		.endif



		.if \hsrr == EXC_HV_OR_STD

		BEGIN_FTR_SECTION

		bne	masked_Hinterrupt

		FTR_SECTION_ELSE

		bne	masked_interrupt

		ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)

		.elseif \hsrr

		bne	masked_Hinterrupt

		.else

		bne	masked_interrupt

		.endif

	.endif



	std	r11,\area\()+EX_R11(r13)

	std	r12,\area\()+EX_R12(r13)



	/*

	 * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI],

	 * because a d-side MCE will clobber those registers so is

	 * not recoverable if they are live.

	 */

	GET_SCRATCH0(r10)

	std	r10,\area\()+EX_R13(r13)

	.if \dar

	.if \hsrr

	mfspr	r10,SPRN_HDAR

	.else

	mfspr	r10,SPRN_DAR

	.endif

	std	r10,\area\()+EX_DAR(r13)

	.endif

	.if \dsisr

	.if \hsrr

	mfspr	r10,SPRN_HDSISR

	.else

	mfspr	r10,SPRN_DSISR

	.endif

	stw	r10,\area\()+EX_DSISR(r13)

	.endif



	.if \early == 2

	/* nothing more */

	.elseif \early

	mfctr	r10			/* save ctr, even for !RELOCATABLE */

	BRANCH_TO_C000(r11, \name\()_early_common)

	.elseif !\virt

	INT_SAVE_SRR_AND_JUMP \name\()_common, \hsrr, \ri

	.else

	INT_VIRT_SAVE_SRR_AND_JUMP \name\()_common, \hsrr

	.endif

	.if \ool

	.popsection

	.endif

.endm



.macro INT_KVM_HANDLER vec, hsrr, area, skip

	.if \hsrr

	TRAMP_KVM_BEGIN(do_kvm_H\vec\())

	.else

	TRAMP_KVM_BEGIN(do_kvm_\vec\())

	.endif

	KVM_HANDLER \vec, \hsrr, \area, \skip

.endm



#define EXC_COMMON(name, realvec, hdlr)					\

	EXC_COMMON_BEGIN(name);						\

	INT_COMMON realvec, PACA_EXGEN, 1, 1, 1, 0, 0 ;			\