Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 99d5040e authored by Max Filippov's avatar Max Filippov Committed by Chris Zankel
Browse files

xtensa: keep a3 and excsave1 on entry to exception handlers 



Based on the SMP patch by Joe Taylor and subsequent fixes.
Preserve exception table pointer (normally stored in excsave1 SR) as it
cannot be easily restored in SMP environment.

Signed-off-by: default avatarMax Filippov <jcmvbkbc@gmail.com>
Signed-off-by: default avatarChris Zankel <chris@zankel.net>
parent 16c5becf

arch/xtensa/include/asm/regs.h

+1 −0
Original line number Diff line number Diff line
@@ -82,6 +82,7 @@ 
#define PS_CALLINC_SHIFT	16

#define PS_CALLINC_MASK		0x00030000

#define PS_OWB_SHIFT		8

#define PS_OWB_WIDTH		4

#define PS_OWB_MASK		0x00000F00

#define PS_RING_SHIFT		6

#define PS_RING_MASK		0x000000C0

arch/xtensa/kernel/align.S

+2 −3
Original line number Diff line number Diff line
@@ -146,9 +146,9 @@ 
 *   a0:	trashed, original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original in DEPC

 *   a3:	dispatch table

 *   a3:	a3

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave_1:	a3

 *   excsave_1:	dispatch table

 *

 *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC

 *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -171,7 +171,6 @@ ENTRY(fast_unaligned) 
	s32i	a8, a2, PT_AREG8



	rsr	a0, depc

	xsr	a3, excsave1

	s32i	a0, a2, PT_AREG2

	s32i	a3, a2, PT_AREG3

arch/xtensa/kernel/coprocessor.S

+4 −5
Original line number Diff line number Diff line
@@ -32,9 +32,9 @@ 
 *   a0:	trashed, original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original in DEPC

 *   a3:	dispatch table

 *   a3:	a3

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave_1:	a3

 *   excsave_1:	dispatch table

 *

 *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC

 *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -225,9 +225,9 @@ ENDPROC(coprocessor_restore) 
 *   a0:	trashed, original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original in DEPC

 *   a3:	dispatch table

 *   a3:	a3

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave_1:	a3

 *   excsave_1:	dispatch table

 *

 *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC

 *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -245,7 +245,6 @@ ENTRY(fast_coprocessor) 


	/* Save remaining registers a1-a3 and SAR */



	xsr	a3, excsave1

	s32i	a3, a2, PT_AREG3

	rsr	a3, sar

	s32i	a1, a2, PT_AREG1

arch/xtensa/kernel/entry.S

+47 −68
Original line number Diff line number Diff line
@@ -91,9 +91,9 @@ 
 *   a0:	trashed, original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original value in depc

 *   a3:	dispatch table

 *   a3:	a3

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave1:	a3

 *   excsave1:	dispatch table

 *

 *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC

 *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -109,9 +109,8 @@ 


ENTRY(user_exception)



	/* Save a2, a3, and depc, restore excsave_1 and set SP. */

	/* Save a1, a2, a3, and set SP. */



	xsr	a3, excsave1

	rsr	a0, depc

	s32i	a1, a2, PT_AREG1

	s32i	a0, a2, PT_AREG2
@@ -237,9 +236,9 @@ ENDPROC(user_exception) 
 *   a0:	trashed, original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original in DEPC

 *   a3:	dispatch table

 *   a3:	a3

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave_1:	a3

 *   excsave_1:	dispatch table

 *

 *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC

 *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -255,9 +254,8 @@ ENDPROC(user_exception) 


ENTRY(kernel_exception)



	/* Save a0, a2, a3, DEPC and set SP. */

	/* Save a1, a2, a3, and set SP. */



	xsr	a3, excsave1		# restore a3, excsave_1

	rsr	a0, depc		# get a2

	s32i	a1, a2, PT_AREG1

	s32i	a0, a2, PT_AREG2
@@ -408,7 +406,7 @@ common_exception: 
	 * exception handler and call the exception handler.

	 */



	movi	a4, exc_table

	rsr	a4, excsave1

	mov	a6, a1			# pass stack frame

	mov	a7, a0			# pass EXCCAUSE

	addx4	a4, a0, a4
@@ -832,9 +830,9 @@ ENDPROC(unrecoverable_exception) 
 *   a0:	trashed, original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original in DEPC

 *   a3:	dispatch table

 *   a3:	a3

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave_1:	a3

 *   excsave_1:	dispatch table

 *

 *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC

 *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -857,18 +855,16 @@ ENTRY(fast_alloca) 


	rsr	a0, depc		# get a2

	s32i	a4, a2, PT_AREG4	# save a4 and

	s32i	a3, a2, PT_AREG3

	s32i	a0, a2, PT_AREG2	# a2 to stack



	/* Exit critical section. */



	movi	a0, 0

	rsr	a3, excsave1

	s32i	a0, a3, EXC_TABLE_FIXUP



	/* Restore a3, excsave_1 */



	xsr	a3, excsave1		# make sure excsave_1 is valid for dbl.

	rsr	a4, epc1		# get exception address

	s32i	a3, a2, PT_AREG3	# save a3 to stack



#ifdef ALLOCA_EXCEPTION_IN_IRAM

#error	iram not supported
@@ -1007,9 +1003,9 @@ ENDPROC(fast_alloca) 
 *   a0:	trashed, original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original in DEPC

 *   a3:	dispatch table

 *   a3:	a3

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave_1:	a3

 *   excsave_1:	dispatch table

 */



ENTRY(fast_syscall_kernel)
@@ -1056,7 +1052,6 @@ ENTRY(fast_syscall_unrecoverable) 


	l32i    a0, a2, PT_AREG0        # restore a0

	xsr     a2, depc                # restore a2, depc

	rsr     a3, excsave1



	wsr     a0, excsave1

	movi    a0, unrecoverable_exception
@@ -1078,10 +1073,10 @@ ENDPROC(fast_syscall_unrecoverable) 
 *   a0:	a2 (syscall-nr), original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original in a0 and DEPC

 *   a3:	dispatch table, original in excsave_1

 *   a3:	a3

 *   a4..a15:	unchanged

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave_1:	a3

 *   excsave_1:	dispatch table

 *

 *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC

 *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -1114,8 +1109,6 @@ ENDPROC(fast_syscall_unrecoverable) 


ENTRY(fast_syscall_xtensa)



	xsr	a3, excsave1		# restore a3, excsave1



	s32i	a7, a2, PT_AREG7	# we need an additional register

	movi	a7, 4			# sizeof(unsigned int)

	access_ok a3, a7, a0, a2, .Leac	# a0: scratch reg, a2: sp
@@ -1178,9 +1171,9 @@ ENDPROC(fast_syscall_xtensa) 
 *   a0:	trashed, original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original in DEPC

 *   a3:	dispatch table

 *   a3:	a3

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave_1:	a3

 *   excsave_1:	dispatch table

 *

 * Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler.

 */
@@ -1189,15 +1182,16 @@ ENTRY(fast_syscall_spill_registers) 


	/* Register a FIXUP handler (pass current wb as a parameter) */



	xsr	a3, excsave1

	movi	a0, fast_syscall_spill_registers_fixup

	s32i	a0, a3, EXC_TABLE_FIXUP

	rsr	a0, windowbase

	s32i	a0, a3, EXC_TABLE_PARAM

	xsr	a3, excsave1		# restore a3 and excsave_1



	/* Save a3 and SAR on stack. */

	/* Save a3, a4 and SAR on stack. */



	rsr	a0, sar

	xsr	a3, excsave1		# restore a3 and excsave_1

	s32i	a3, a2, PT_AREG3

	s32i	a4, a2, PT_AREG4

	s32i	a0, a2, PT_AREG5	# store SAR to PT_AREG5
@@ -1251,14 +1245,14 @@ fast_syscall_spill_registers_fixup: 
	 * in WS, so that the exception handlers save them to the task stack.

	 */



	rsr	a3, excsave1	# get spill-mask

	xsr	a3, excsave1	# get spill-mask

	slli	a2, a3, 1	# shift left by one



	slli	a3, a2, 32-WSBITS

	src	a2, a2, a3	# a1 = xxwww1yyxxxwww1yy......

	wsr	a2, windowstart	# set corrected windowstart



	movi	a3, exc_table

	rsr	a3, excsave1

	l32i	a2, a3, EXC_TABLE_DOUBLE_SAVE	# restore a2

	l32i	a3, a3, EXC_TABLE_PARAM	# original WB (in user task)
@@ -1295,7 +1289,7 @@ fast_syscall_spill_registers_fixup: 


	/* Jump to the exception handler. */



	movi	a3, exc_table

	rsr	a3, excsave1

	rsr	a0, exccause

	addx4	a0, a0, a3              	# find entry in table

	l32i	a0, a0, EXC_TABLE_FAST_USER     # load handler
@@ -1312,6 +1306,7 @@ fast_syscall_spill_registers_fixup_return: 
	xsr	a3, excsave1

	movi	a2, fast_syscall_spill_registers_fixup

	s32i	a2, a3, EXC_TABLE_FIXUP

	s32i	a0, a3, EXC_TABLE_DOUBLE_SAVE

	rsr	a2, windowbase

	s32i	a2, a3, EXC_TABLE_PARAM

	l32i	a2, a3, EXC_TABLE_KSTK
@@ -1323,11 +1318,6 @@ fast_syscall_spill_registers_fixup_return: 
	wsr	a3, windowbase

	rsync



	/* Restore a3 and return. */



	movi	a3, exc_table

	xsr	a3, excsave1



	rfde
@@ -1514,9 +1504,8 @@ ENTRY(_spill_registers) 


	movi	a0, 0



	movi	a3, exc_table

	rsr	a3, excsave1

	l32i	a1, a3, EXC_TABLE_KSTK

	wsr	a3, excsave1



	movi	a4, (1 << PS_WOE_BIT) | LOCKLEVEL

	wsr	a4, ps
@@ -1560,9 +1549,9 @@ ENDPROC(fast_second_level_miss_double_kernel) 
 *   a0:	trashed, original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original in DEPC

 *   a3:	dispatch table

 *   a3:	a3

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave_1:	a3

 *   excsave_1:	dispatch table

 *

 *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC

 *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -1570,9 +1559,10 @@ ENDPROC(fast_second_level_miss_double_kernel) 


ENTRY(fast_second_level_miss)



	/* Save a1. Note: we don't expect a double exception. */

	/* Save a1 and a3. Note: we don't expect a double exception. */



	s32i	a1, a2, PT_AREG1

	s32i	a3, a2, PT_AREG3



	/* We need to map the page of PTEs for the user task.  Find

	 * the pointer to that page.  Also, it's possible for tsk->mm
@@ -1594,9 +1584,6 @@ ENTRY(fast_second_level_miss) 
	l32i	a0, a1, TASK_MM		# tsk->mm

	beqz	a0, 9f





	/* We deliberately destroy a3 that holds the exception table. */



8:	rsr	a3, excvaddr		# fault address

	_PGD_OFFSET(a0, a3, a1)

	l32i	a0, a0, 0		# read pmdval
@@ -1647,7 +1634,7 @@ ENTRY(fast_second_level_miss) 


	/* Exit critical section. */



4:	movi	a3, exc_table		# restore a3

4:	rsr	a3, excsave1

	movi	a0, 0

	s32i	a0, a3, EXC_TABLE_FIXUP
@@ -1655,8 +1642,8 @@ ENTRY(fast_second_level_miss) 


	l32i	a0, a2, PT_AREG0

	l32i	a1, a2, PT_AREG1

	l32i	a3, a2, PT_AREG3

	l32i	a2, a2, PT_DEPC

	xsr	a3, excsave1



	bgeui	a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
@@ -1743,11 +1730,8 @@ ENTRY(fast_second_level_miss) 


2:	/* Invalid PGD, default exception handling */



	movi	a3, exc_table

	rsr	a1, depc

	xsr	a3, excsave1

	s32i	a1, a2, PT_AREG2

	s32i	a3, a2, PT_AREG3

	mov	a1, a2



	rsr	a2, ps
@@ -1767,9 +1751,9 @@ ENDPROC(fast_second_level_miss) 
 *   a0:	trashed, original value saved on stack (PT_AREG0)

 *   a1:	a1

 *   a2:	new stack pointer, original in DEPC

 *   a3:	dispatch table

 *   a3:	a3

 *   depc:	a2, original value saved on stack (PT_DEPC)

 *   excsave_1:	a3

 *   excsave_1:	dispatch table

 *

 *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC

 *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
@@ -1777,17 +1761,17 @@ ENDPROC(fast_second_level_miss) 


ENTRY(fast_store_prohibited)



	/* Save a1 and a4. */

	/* Save a1 and a3. */



	s32i	a1, a2, PT_AREG1

	s32i	a4, a2, PT_AREG4

	s32i	a3, a2, PT_AREG3



	GET_CURRENT(a1,a2)

	l32i	a0, a1, TASK_MM		# tsk->mm

	beqz	a0, 9f



8:	rsr	a1, excvaddr		# fault address

	_PGD_OFFSET(a0, a1, a4)

	_PGD_OFFSET(a0, a1, a3)

	l32i	a0, a0, 0

	beqz	a0, 2f
@@ -1796,39 +1780,37 @@ ENTRY(fast_store_prohibited) 
	 * and is not PAGE_NONE. See pgtable.h for possible PTE layouts.

	 */



	_PTE_OFFSET(a0, a1, a4)

	l32i	a4, a0, 0		# read pteval

	_PTE_OFFSET(a0, a1, a3)

	l32i	a3, a0, 0		# read pteval

	movi	a1, _PAGE_CA_INVALID

	ball	a4, a1, 2f

	bbci.l	a4, _PAGE_WRITABLE_BIT, 2f

	ball	a3, a1, 2f

	bbci.l	a3, _PAGE_WRITABLE_BIT, 2f



	movi	a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE

	or	a4, a4, a1

	or	a3, a3, a1

	rsr	a1, excvaddr

	s32i	a4, a0, 0

	s32i	a3, a0, 0



	/* We need to flush the cache if we have page coloring. */

#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK

	dhwb	a0, 0

#endif

	pdtlb	a0, a1

	wdtlb	a4, a0

	wdtlb	a3, a0



	/* Exit critical section. */



	movi	a0, 0

	rsr	a3, excsave1

	s32i	a0, a3, EXC_TABLE_FIXUP



	/* Restore the working registers, and return. */



	l32i	a4, a2, PT_AREG4

	l32i	a3, a2, PT_AREG3

	l32i	a1, a2, PT_AREG1

	l32i	a0, a2, PT_AREG0

	l32i	a2, a2, PT_DEPC



	/* Restore excsave1 and a3. */



	xsr	a3, excsave1

	bgeui	a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f



	rsr	a2, depc
@@ -1845,11 +1827,8 @@ ENTRY(fast_store_prohibited) 


2:	/* If there was a problem, handle fault in C */



	rsr	a4, depc	# still holds a2

	xsr	a3, excsave1

	s32i	a4, a2, PT_AREG2

	s32i	a3, a2, PT_AREG3

	l32i	a4, a2, PT_AREG4

	rsr	a3, depc	# still holds a2

	s32i	a3, a2, PT_AREG2

	mov	a1, a2



	rsr	a2, ps

arch/xtensa/kernel/vectors.S

+184 −66
Original line number Diff line number Diff line
@@ -78,6 +78,7 @@ ENTRY(_UserExceptionVector) 
	s32i	a0, a2, PT_DEPC		# mark it as a regular exception

	addx4	a0, a0, a3		# find entry in table

	l32i	a0, a0, EXC_TABLE_FAST_USER	# load handler

	xsr	a3, excsave1		# restore a3 and dispatch table

	jx	a0



ENDPROC(_UserExceptionVector)
@@ -104,6 +105,7 @@ ENTRY(_KernelExceptionVector) 
	s32i	a0, a2, PT_DEPC		# mark it as a regular exception

	addx4	a0, a0, a3		# find entry in table

	l32i	a0, a0, EXC_TABLE_FAST_KERNEL	# load handler address

	xsr	a3, excsave1		# restore a3 and dispatch table

	jx	a0



ENDPROC(_KernelExceptionVector)
@@ -168,7 +170,7 @@ ENDPROC(_KernelExceptionVector) 
 *

 *	a0:	   DEPC

 *	a1: 	   a1

 *	a2:	   trashed, original value in EXC_TABLE_DOUBLE_A2

 *	a2:	   trashed, original value in EXC_TABLE_DOUBLE_SAVE

 *	a3:	   exctable

 *	depc:	   a0

 *	excsave_1: a3
@@ -204,47 +206,46 @@ ENDPROC(_KernelExceptionVector) 


	.section .DoubleExceptionVector.text, "ax"

	.begin literal_prefix .DoubleExceptionVector

	.globl _DoubleExceptionVector_WindowUnderflow

	.globl _DoubleExceptionVector_WindowOverflow



ENTRY(_DoubleExceptionVector)



	/* Deliberately destroy excsave (don't assume it's value was valid). */



	wsr	a3, excsave1		# save a3

	xsr	a3, excsave1

	s32i	a2, a3, EXC_TABLE_DOUBLE_SAVE



	/* Check for kernel double exception (usually fatal). */



	rsr	a3, ps

	_bbci.l	a3, PS_UM_BIT, .Lksp

	rsr	a2, ps

	_bbci.l	a2, PS_UM_BIT, .Lksp



	/* Check if we are currently handling a window exception. */

	/* Note: We don't need to indicate that we enter a critical section. */



	xsr	a0, depc		# get DEPC, save a0



	movi	a3, WINDOW_VECTORS_VADDR

	_bltu	a0, a3, .Lfixup

	addi	a3, a3, WINDOW_VECTORS_SIZE

	_bgeu	a0, a3, .Lfixup

	movi	a2, WINDOW_VECTORS_VADDR

	_bltu	a0, a2, .Lfixup

	addi	a2, a2, WINDOW_VECTORS_SIZE

	_bgeu	a0, a2, .Lfixup



	/* Window overflow/underflow exception. Get stack pointer. */



	mov	a3, a2

	/* This explicit literal and the following references to it are made

	 * in order to fit DoubleExceptionVector.literals into the available

	 * 16-byte gap before DoubleExceptionVector.text in the absence of

	 * link time relaxation. See kernel/vmlinux.lds.S

	 */

	.literal .Lexc_table, exc_table

	l32r	a2, .Lexc_table

	l32i	a2, a2, EXC_TABLE_KSTK

	l32i	a2, a3, EXC_TABLE_KSTK



	/* Check for overflow/underflow exception, jump if overflow. */



	_bbci.l	a0, 6, .Lovfl

	_bbci.l	a0, 6, _DoubleExceptionVector_WindowOverflow



	/* a0: depc, a1: a1, a2: kstk, a3: a2, depc: a0, excsave: a3  */



	/* Restart window underflow exception.

	/*

	 * Restart window underflow exception.

	 * Currently:

	 *	depc = orig a0,

	 *	a0 = orig DEPC,

	 *	a2 = new sp based on KSTK from exc_table

	 *	a3 = excsave_1

	 *	excsave_1 = orig a3

	 *

	 * We return to the instruction in user space that caused the window

	 * underflow exception. Therefore, we change window base to the value

	 * before we entered the window underflow exception and prepare the
@@ -252,10 +253,11 @@ ENTRY(_DoubleExceptionVector) 
	 * by changing depc (in a0).

	 * Note: We can trash the current window frame (a0...a3) and depc!

	 */



_DoubleExceptionVector_WindowUnderflow:

	xsr	a3, excsave1

	wsr	a2, depc		# save stack pointer temporarily

	rsr	a0, ps

	extui	a0, a0, PS_OWB_SHIFT, 4

	extui	a0, a0, PS_OWB_SHIFT, PS_OWB_WIDTH

	wsr	a0, windowbase

	rsync
@@ -263,28 +265,57 @@ ENTRY(_DoubleExceptionVector) 


	xsr	a2, depc		# save a2 and get stack pointer

	s32i	a0, a2, PT_AREG0



	wsr	a3, excsave1		# save a3

	l32r	a3, .Lexc_table



	xsr	a3, excsave1

	rsr	a0, exccause

	s32i	a0, a2, PT_DEPC		# mark it as a regular exception

	addx4	a0, a0, a3

	xsr	a3, excsave1

	l32i	a0, a0, EXC_TABLE_FAST_USER

	jx	a0



.Lfixup:/* Check for a fixup handler or if we were in a critical section. */

	/*

	 * We only allow the ITLB miss exception if we are in kernel space.

	 * All other exceptions are unexpected and thus unrecoverable!

	 */



#ifdef CONFIG_MMU

	.extern fast_second_level_miss_double_kernel



.Lksp:	/* a0: a0, a1: a1, a2: a2, a3: trashed, depc: depc, excsave: a3 */



	rsr	a3, exccause

	beqi	a3, EXCCAUSE_ITLB_MISS, 1f

	addi	a3, a3, -EXCCAUSE_DTLB_MISS

	bnez	a3, .Lunrecoverable

1:	movi	a3, fast_second_level_miss_double_kernel

	jx	a3

#else

.equ	.Lksp,	.Lunrecoverable

#endif



	/* a0: depc, a1: a1, a2: a2, a3: trashed, depc: a0, excsave1: a3 */

	/* Critical! We can't handle this situation. PANIC! */



	l32r	a3, .Lexc_table

	s32i	a2, a3, EXC_TABLE_DOUBLE_SAVE	# temporary variable

	.extern unrecoverable_exception



.Lunrecoverable_fixup:

	l32i	a2, a3, EXC_TABLE_DOUBLE_SAVE

	xsr	a0, depc



.Lunrecoverable:

	rsr	a3, excsave1

	wsr	a0, excsave1

	movi	a0, unrecoverable_exception

	callx0	a0



.Lfixup:/* Check for a fixup handler or if we were in a critical section. */



	/* a0: depc, a1: a1, a2: trash, a3: exctable, depc: a0, excsave1: a3 */



	/* Enter critical section. */



	l32i	a2, a3, EXC_TABLE_FIXUP

	s32i	a3, a3, EXC_TABLE_FIXUP

	beq	a2, a3, .Lunrecoverable_fixup	# critical!

	beq	a2, a3, .Lunrecoverable_fixup	# critical section

	beqz	a2, .Ldflt			# no handler was registered



	/* a0: depc, a1: a1, a2: trash, a3: exctable, depc: a0, excsave: a3 */
@@ -293,58 +324,145 @@ ENTRY(_DoubleExceptionVector) 


.Ldflt:	/* Get stack pointer. */



	l32i	a3, a3, EXC_TABLE_DOUBLE_SAVE

	addi	a2, a3, -PT_USER_SIZE



.Lovfl:	/* Jump to default handlers. */

	l32i	a2, a3, EXC_TABLE_DOUBLE_SAVE

	addi	a2, a2, -PT_USER_SIZE



	/* a0: depc, a1: a1, a2: kstk, a3: a2, depc: a0, excsave: a3 */

	/* a0: depc, a1: a1, a2: kstk, a3: exctable, depc: a0, excsave: a3 */



	xsr	a3, depc

	s32i	a0, a2, PT_DEPC

	s32i	a3, a2, PT_AREG0

	l32i	a0, a3, EXC_TABLE_DOUBLE_SAVE

	xsr	a0, depc

	s32i	a0, a2, PT_AREG0



	/* a0: avail, a1: a1, a2: kstk, a3: avail, depc: a2, excsave: a3 */

	/* a0: avail, a1: a1, a2: kstk, a3: exctable, depc: a2, excsave: a3 */



	l32r	a3, .Lexc_table

	rsr	a0, exccause

	addx4	a0, a0, a3

	xsr	a3, excsave1

	l32i	a0, a0, EXC_TABLE_FAST_USER

	jx	a0



	/*

	 * We only allow the ITLB miss exception if we are in kernel space.

	 * All other exceptions are unexpected and thus unrecoverable!

	 * Restart window OVERFLOW exception.

	 * Currently:

	 *	depc = orig a0,

	 *	a0 = orig DEPC,

	 *	a2 = new sp based on KSTK from exc_table

	 *	a3 = EXCSAVE_1

	 *	excsave_1 = orig a3

	 *

	 * We return to the instruction in user space that caused the window

	 * overflow exception. Therefore, we change window base to the value

	 * before we entered the window overflow exception and prepare the

	 * registers to return as if we were coming from a regular exception

	 * by changing DEPC (in a0).

	 *

	 * NOTE: We CANNOT trash the current window frame (a0...a3), but we

	 * can clobber depc.

	 *

	 * The tricky part here is that overflow8 and overflow12 handlers

	 * save a0, then clobber a0.  To restart the handler, we have to restore

	 * a0 if the double exception was past the point where a0 was clobbered.

	 *

	 * To keep things simple, we take advantage of the fact all overflow

	 * handlers save a0 in their very first instruction.  If DEPC was past

	 * that instruction, we can safely restore a0 from where it was saved

	 * on the stack.

	 *

	 * a0: depc, a1: a1, a2: kstk, a3: exc_table, depc: a0, excsave1: a3

	 */

_DoubleExceptionVector_WindowOverflow:

	extui	a2, a0, 0, 6	# get offset into 64-byte vector handler

	beqz	a2, 1f		# if at start of vector, don't restore



#ifdef CONFIG_MMU

	.extern fast_second_level_miss_double_kernel

	addi	a0, a0, -128

	bbsi	a0, 8, 1f	# don't restore except for overflow 8 and 12

	bbsi	a0, 7, 2f



.Lksp:	/* a0: a0, a1: a1, a2: a2, a3: trashed, depc: depc, excsave: a3 */

	/*

	 * Restore a0 as saved by _WindowOverflow8().

	 *

	 * FIXME:  we really need a fixup handler for this L32E,

	 * for the extremely unlikely case where the overflow handler's

	 * reference thru a0 gets a hardware TLB refill that bumps out

	 * the (distinct, aliasing) TLB entry that mapped its prior

	 * references thru a9, and where our reference now thru a9

	 * gets a 2nd-level miss exception (not hardware TLB refill).

	 */



	rsr	a3, exccause

	beqi	a3, EXCCAUSE_ITLB_MISS, 1f

	addi	a3, a3, -EXCCAUSE_DTLB_MISS

	bnez	a3, .Lunrecoverable

1:	movi	a3, fast_second_level_miss_double_kernel

	jx	a3

#else

.equ	.Lksp,	.Lunrecoverable

#endif

	l32e	a2, a9, -16

	wsr	a2, depc	# replace the saved a0

	j	1f



	/* Critical! We can't handle this situation. PANIC! */

2:

	/*

	 * Restore a0 as saved by _WindowOverflow12().

	 *

	 * FIXME:  we really need a fixup handler for this L32E,

	 * for the extremely unlikely case where the overflow handler's

	 * reference thru a0 gets a hardware TLB refill that bumps out

	 * the (distinct, aliasing) TLB entry that mapped its prior

	 * references thru a13, and where our reference now thru a13

	 * gets a 2nd-level miss exception (not hardware TLB refill).

	 */



	.extern unrecoverable_exception

	l32e	a2, a13, -16

	wsr	a2, depc	# replace the saved a0

1:

	/*

	 * Restore WindowBase while leaving all address registers restored.

	 * We have to use ROTW for this, because WSR.WINDOWBASE requires

	 * an address register (which would prevent restore).

	 *

	 * Window Base goes from 0 ... 7 (Module 8)

	 * Window Start is 8 bits; Ex: (0b1010 1010):0x55 from series of call4s

	 */



	rsr	a0, ps

	extui	a0, a0, PS_OWB_SHIFT, PS_OWB_WIDTH

	rsr	a2, windowbase

	sub	a0, a2, a0

	extui	a0, a0, 0, 3



.Lunrecoverable_fixup:

	l32i	a2, a3, EXC_TABLE_DOUBLE_SAVE

	xsr	a0, depc

	xsr	a3, excsave1

	beqi	a0, 1, .L1pane

	beqi	a0, 3, .L3pane



.Lunrecoverable:

	rsr	a3, excsave1

	wsr	a0, excsave1

	movi	a0, unrecoverable_exception

	callx0	a0

	rsr	a0, depc

	rotw	-2



	/*

	 * We are now in the user code's original window frame.

	 * Process the exception as a user exception as if it was

	 * taken by the user code.

	 *

	 * This is similar to the user exception vector,

	 * except that PT_DEPC isn't set to EXCCAUSE.

	 */

1:

	xsr	a3, excsave1

	wsr	a2, depc

	l32i	a2, a3, EXC_TABLE_KSTK

	s32i	a0, a2, PT_AREG0

	rsr	a0, exccause



	s32i	a0, a2, PT_DEPC



	addx4	a0, a0, a3

	l32i	a0, a0, EXC_TABLE_FAST_USER

	xsr	a3, excsave1

	jx	a0



.L1pane:

	rsr	a0, depc

	rotw	-1

	j	1b



.L3pane:

	rsr	a0, depc

	rotw	-3

	j	1b



	.end literal_prefix