Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2ab82e66 authored by Markos Chandras's avatar Markos Chandras Committed by Ralf Baechle
Browse files

MIPS: lib: csum_partial: Merge EXC and load/store macros 



Each load/store macro always adds an entry to the __ex_table
using the EXC macro. There are cases where a load instruction may
never fail such as when we are sure the load happens in the kernel
address space. Therefore, we merge these the EXC and LOADX/STOREX
macros into a single one. We also expand the argument list in the EXC
macro to make the macro more flexible. The extra 'type' argument is not
used by this commit, but it will be used when EVA support is added to
memcpy.

Signed-off-by: default avatarMarkos Chandras <markos.chandras@imgtec.com>
parent ac85227f

arch/mips/lib/csum_partial.S

+91 −69
Original line number Original line Diff line number Diff line
@@ -328,20 +328,39 @@ LEAF(csum_partial) 
 * These handlers do not need to overwrite any data.

 * These handlers do not need to overwrite any data.

 */

 */





#define EXC(inst_reg,addr,handler)		\

/* Instruction type */

9:	inst_reg, addr;				\

#define LD_INSN 1

#define ST_INSN 2



/*

 * Wrapper to add an entry in the exception table

 * in case the insn causes a memory exception.

 * Arguments:

 * insn    : Load/store instruction

 * type    : Instruction type

 * reg     : Register

 * addr    : Address

 * handler : Exception handler

 */

#define EXC(insn, type, reg, addr, handler)	\

9:	insn reg, addr;				\

	.section __ex_table,"a";		\

	.section __ex_table,"a";		\

	PTR	9b, handler;			\

	PTR	9b, handler;			\

	.previous

	.previous





#undef LOAD



#ifdef USE_DOUBLE

#ifdef USE_DOUBLE





#define LOAD   ld

#define LOADK	ld /* No exception */

#define LOADL  ldl

#define LOAD(reg, addr, handler)	EXC(ld, LD_INSN, reg, addr, handler)

#define LOADR  ldr

#define LOADBU(reg, addr, handler)	EXC(lbu, LD_INSN, reg, addr, handler)

#define STOREL sdl

#define LOADL(reg, addr, handler)	EXC(ldl, LD_INSN, reg, addr, handler)

#define STORER sdr

#define LOADR(reg, addr, handler)	EXC(ldr, LD_INSN, reg, addr, handler)

#define STORE  sd

#define STOREB(reg, addr, handler)	EXC(sb, ST_INSN, reg, addr, handler)

#define STOREL(reg, addr, handler)	EXC(sdl, ST_INSN, reg, addr, handler)

#define STORER(reg, addr, handler)	EXC(sdr, ST_INSN, reg, addr, handler)

#define STORE(reg, addr, handler)	EXC(sd, ST_INSN, reg, addr, handler)

#define ADD    daddu

#define ADD    daddu

#define SUB    dsubu

#define SUB    dsubu

#define SRL    dsrl

#define SRL    dsrl
@@ -353,12 +372,15 @@ LEAF(csum_partial) 




#else

#else





#define LOAD   lw

#define LOADK	lw /* No exception */

#define LOADL  lwl

#define LOAD(reg, addr, handler)	EXC(lw, LD_INSN, reg, addr, handler)

#define LOADR  lwr

#define LOADBU(reg, addr, handler)	EXC(lbu, LD_INSN, reg, addr, handler)

#define STOREL swl

#define LOADL(reg, addr, handler)	EXC(lwl, LD_INSN, reg, addr, handler)

#define STORER swr

#define LOADR(reg, addr, handler)	EXC(lwr, LD_INSN, reg, addr, handler)

#define STORE  sw

#define STOREB(reg, addr, handler)	EXC(sb, ST_INSN, reg, addr, handler)

#define STOREL(reg, addr, handler)	EXC(swl, ST_INSN, reg, addr, handler)

#define STORER(reg, addr, handler)	EXC(swr, ST_INSN, reg, addr, handler)

#define STORE(reg, addr, handler)	EXC(sw, ST_INSN, reg, addr, handler)

#define ADD    addu

#define ADD    addu

#define SUB    subu

#define SUB    subu

#define SRL    srl

#define SRL    srl
@@ -439,31 +461,31 @@ FEXPORT(csum_partial_copy_nocheck) 
	SUB	len, 8*NBYTES		# subtract here for bgez loop

	SUB	len, 8*NBYTES		# subtract here for bgez loop

	.align	4

	.align	4

1:

1:

EXC(	LOAD	t0, UNIT(0)(src),	.Ll_exc)

	LOAD(t0, UNIT(0)(src), .Ll_exc)

EXC(	LOAD	t1, UNIT(1)(src),	.Ll_exc_copy)

	LOAD(t1, UNIT(1)(src), .Ll_exc_copy)

EXC(	LOAD	t2, UNIT(2)(src),	.Ll_exc_copy)

	LOAD(t2, UNIT(2)(src), .Ll_exc_copy)

EXC(	LOAD	t3, UNIT(3)(src),	.Ll_exc_copy)

	LOAD(t3, UNIT(3)(src), .Ll_exc_copy)

EXC(	LOAD	t4, UNIT(4)(src),	.Ll_exc_copy)

	LOAD(t4, UNIT(4)(src), .Ll_exc_copy)

EXC(	LOAD	t5, UNIT(5)(src),	.Ll_exc_copy)

	LOAD(t5, UNIT(5)(src), .Ll_exc_copy)

EXC(	LOAD	t6, UNIT(6)(src),	.Ll_exc_copy)

	LOAD(t6, UNIT(6)(src), .Ll_exc_copy)

EXC(	LOAD	t7, UNIT(7)(src),	.Ll_exc_copy)

	LOAD(t7, UNIT(7)(src), .Ll_exc_copy)

	SUB	len, len, 8*NBYTES

	SUB	len, len, 8*NBYTES

	ADD	src, src, 8*NBYTES

	ADD	src, src, 8*NBYTES

EXC(	STORE	t0, UNIT(0)(dst),	.Ls_exc)

	STORE(t0, UNIT(0)(dst),	.Ls_exc)

	ADDC(sum, t0)

	ADDC(sum, t0)

EXC(	STORE	t1, UNIT(1)(dst),	.Ls_exc)

	STORE(t1, UNIT(1)(dst),	.Ls_exc)

	ADDC(sum, t1)

	ADDC(sum, t1)

EXC(	STORE	t2, UNIT(2)(dst),	.Ls_exc)

	STORE(t2, UNIT(2)(dst),	.Ls_exc)

	ADDC(sum, t2)

	ADDC(sum, t2)

EXC(	STORE	t3, UNIT(3)(dst),	.Ls_exc)

	STORE(t3, UNIT(3)(dst),	.Ls_exc)

	ADDC(sum, t3)

	ADDC(sum, t3)

EXC(	STORE	t4, UNIT(4)(dst),	.Ls_exc)

	STORE(t4, UNIT(4)(dst),	.Ls_exc)

	ADDC(sum, t4)

	ADDC(sum, t4)

EXC(	STORE	t5, UNIT(5)(dst),	.Ls_exc)

	STORE(t5, UNIT(5)(dst),	.Ls_exc)

	ADDC(sum, t5)

	ADDC(sum, t5)

EXC(	STORE	t6, UNIT(6)(dst),	.Ls_exc)

	STORE(t6, UNIT(6)(dst),	.Ls_exc)

	ADDC(sum, t6)

	ADDC(sum, t6)

EXC(	STORE	t7, UNIT(7)(dst),	.Ls_exc)

	STORE(t7, UNIT(7)(dst),	.Ls_exc)

	ADDC(sum, t7)

	ADDC(sum, t7)

	.set	reorder				/* DADDI_WAR */

	.set	reorder				/* DADDI_WAR */

	ADD	dst, dst, 8*NBYTES

	ADD	dst, dst, 8*NBYTES
@@ -483,19 +505,19 @@ EXC( STORE t7, UNIT(7)(dst), .Ls_exc) 
	/*

	/*

	 * len >= 4*NBYTES

	 * len >= 4*NBYTES

	 */

	 */

EXC(	LOAD	t0, UNIT(0)(src),	.Ll_exc)

	LOAD(t0, UNIT(0)(src), .Ll_exc)

EXC(	LOAD	t1, UNIT(1)(src),	.Ll_exc_copy)

	LOAD(t1, UNIT(1)(src), .Ll_exc_copy)

EXC(	LOAD	t2, UNIT(2)(src),	.Ll_exc_copy)

	LOAD(t2, UNIT(2)(src), .Ll_exc_copy)

EXC(	LOAD	t3, UNIT(3)(src),	.Ll_exc_copy)

	LOAD(t3, UNIT(3)(src), .Ll_exc_copy)

	SUB	len, len, 4*NBYTES

	SUB	len, len, 4*NBYTES

	ADD	src, src, 4*NBYTES

	ADD	src, src, 4*NBYTES

EXC(	STORE	t0, UNIT(0)(dst),	.Ls_exc)

	STORE(t0, UNIT(0)(dst),	.Ls_exc)

	ADDC(sum, t0)

	ADDC(sum, t0)

EXC(	STORE	t1, UNIT(1)(dst),	.Ls_exc)

	STORE(t1, UNIT(1)(dst),	.Ls_exc)

	ADDC(sum, t1)

	ADDC(sum, t1)

EXC(	STORE	t2, UNIT(2)(dst),	.Ls_exc)

	STORE(t2, UNIT(2)(dst),	.Ls_exc)

	ADDC(sum, t2)

	ADDC(sum, t2)

EXC(	STORE	t3, UNIT(3)(dst),	.Ls_exc)

	STORE(t3, UNIT(3)(dst),	.Ls_exc)

	ADDC(sum, t3)

	ADDC(sum, t3)

	.set	reorder				/* DADDI_WAR */

	.set	reorder				/* DADDI_WAR */

	ADD	dst, dst, 4*NBYTES

	ADD	dst, dst, 4*NBYTES
@@ -508,10 +530,10 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 
	beq	rem, len, .Lcopy_bytes

	beq	rem, len, .Lcopy_bytes

	 nop

	 nop

1:

1:

EXC(	LOAD	t0, 0(src),		.Ll_exc)

	LOAD(t0, 0(src), .Ll_exc)

	ADD	src, src, NBYTES

	ADD	src, src, NBYTES

	SUB	len, len, NBYTES

	SUB	len, len, NBYTES

EXC(	STORE	t0, 0(dst),		.Ls_exc)

	STORE(t0, 0(dst), .Ls_exc)

	ADDC(sum, t0)

	ADDC(sum, t0)

	.set	reorder				/* DADDI_WAR */

	.set	reorder				/* DADDI_WAR */

	ADD	dst, dst, NBYTES

	ADD	dst, dst, NBYTES
@@ -534,10 +556,10 @@ EXC( STORE t0, 0(dst), .Ls_exc) 
	 ADD	t1, dst, len	# t1 is just past last byte of dst

	 ADD	t1, dst, len	# t1 is just past last byte of dst

	li	bits, 8*NBYTES

	li	bits, 8*NBYTES

	SLL	rem, len, 3	# rem = number of bits to keep

	SLL	rem, len, 3	# rem = number of bits to keep

EXC(	LOAD	t0, 0(src),		.Ll_exc)

	LOAD(t0, 0(src), .Ll_exc)

	SUB	bits, bits, rem # bits = number of bits to discard

	SUB	bits, bits, rem # bits = number of bits to discard

	SHIFT_DISCARD t0, t0, bits

	SHIFT_DISCARD t0, t0, bits

EXC(	STREST	t0, -1(t1),		.Ls_exc)

	STREST(t0, -1(t1), .Ls_exc)

	SHIFT_DISCARD_REVERT t0, t0, bits

	SHIFT_DISCARD_REVERT t0, t0, bits

	.set reorder

	.set reorder

	ADDC(sum, t0)

	ADDC(sum, t0)
@@ -554,12 +576,12 @@ EXC( STREST t0, -1(t1), .Ls_exc) 
	 * Set match = (src and dst have same alignment)

	 * Set match = (src and dst have same alignment)

	 */

	 */

#define match rem

#define match rem

EXC(	LDFIRST t3, FIRST(0)(src),	.Ll_exc)

	LDFIRST(t3, FIRST(0)(src), .Ll_exc)

	ADD	t2, zero, NBYTES

	ADD	t2, zero, NBYTES

EXC(	LDREST	t3, REST(0)(src),	.Ll_exc_copy)

	LDREST(t3, REST(0)(src), .Ll_exc_copy)

	SUB	t2, t2, t1	# t2 = number of bytes copied

	SUB	t2, t2, t1	# t2 = number of bytes copied

	xor	match, t0, t1

	xor	match, t0, t1

EXC(	STFIRST t3, FIRST(0)(dst),	.Ls_exc)

	STFIRST(t3, FIRST(0)(dst), .Ls_exc)

	SLL	t4, t1, 3		# t4 = number of bits to discard

	SLL	t4, t1, 3		# t4 = number of bits to discard

	SHIFT_DISCARD t3, t3, t4

	SHIFT_DISCARD t3, t3, t4

	/* no SHIFT_DISCARD_REVERT to handle odd buffer properly */

	/* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
@@ -581,26 +603,26 @@ EXC( STFIRST t3, FIRST(0)(dst), .Ls_exc) 
 * It's OK to load FIRST(N+1) before REST(N) because the two addresses

 * It's OK to load FIRST(N+1) before REST(N) because the two addresses

 * are to the same unit (unless src is aligned, but it's not).

 * are to the same unit (unless src is aligned, but it's not).

 */

 */

EXC(	LDFIRST t0, FIRST(0)(src),	.Ll_exc)

	LDFIRST(t0, FIRST(0)(src), .Ll_exc)

EXC(	LDFIRST t1, FIRST(1)(src),	.Ll_exc_copy)

	LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy)

	SUB	len, len, 4*NBYTES

	SUB	len, len, 4*NBYTES

EXC(	LDREST	t0, REST(0)(src),	.Ll_exc_copy)

	LDREST(t0, REST(0)(src), .Ll_exc_copy)

EXC(	LDREST	t1, REST(1)(src),	.Ll_exc_copy)

	LDREST(t1, REST(1)(src), .Ll_exc_copy)

EXC(	LDFIRST t2, FIRST(2)(src),	.Ll_exc_copy)

	LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy)

EXC(	LDFIRST t3, FIRST(3)(src),	.Ll_exc_copy)

	LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy)

EXC(	LDREST	t2, REST(2)(src),	.Ll_exc_copy)

	LDREST(t2, REST(2)(src), .Ll_exc_copy)

EXC(	LDREST	t3, REST(3)(src),	.Ll_exc_copy)

	LDREST(t3, REST(3)(src), .Ll_exc_copy)

	ADD	src, src, 4*NBYTES

	ADD	src, src, 4*NBYTES

#ifdef CONFIG_CPU_SB1

#ifdef CONFIG_CPU_SB1

	nop				# improves slotting

	nop				# improves slotting

#endif

#endif

EXC(	STORE	t0, UNIT(0)(dst),	.Ls_exc)

	STORE(t0, UNIT(0)(dst),	.Ls_exc)

	ADDC(sum, t0)

	ADDC(sum, t0)

EXC(	STORE	t1, UNIT(1)(dst),	.Ls_exc)

	STORE(t1, UNIT(1)(dst),	.Ls_exc)

	ADDC(sum, t1)

	ADDC(sum, t1)

EXC(	STORE	t2, UNIT(2)(dst),	.Ls_exc)

	STORE(t2, UNIT(2)(dst),	.Ls_exc)

	ADDC(sum, t2)

	ADDC(sum, t2)

EXC(	STORE	t3, UNIT(3)(dst),	.Ls_exc)

	STORE(t3, UNIT(3)(dst),	.Ls_exc)

	ADDC(sum, t3)

	ADDC(sum, t3)

	.set	reorder				/* DADDI_WAR */

	.set	reorder				/* DADDI_WAR */

	ADD	dst, dst, 4*NBYTES

	ADD	dst, dst, 4*NBYTES
@@ -613,11 +635,11 @@ EXC( STORE t3, UNIT(3)(dst), .Ls_exc) 
	beq	rem, len, .Lcopy_bytes

	beq	rem, len, .Lcopy_bytes

	 nop

	 nop

1:

1:

EXC(	LDFIRST t0, FIRST(0)(src),	.Ll_exc)

	LDFIRST(t0, FIRST(0)(src), .Ll_exc)

EXC(	LDREST	t0, REST(0)(src),	.Ll_exc_copy)

	LDREST(t0, REST(0)(src), .Ll_exc_copy)

	ADD	src, src, NBYTES

	ADD	src, src, NBYTES

	SUB	len, len, NBYTES

	SUB	len, len, NBYTES

EXC(	STORE	t0, 0(dst),		.Ls_exc)

	STORE(t0, 0(dst), .Ls_exc)

	ADDC(sum, t0)

	ADDC(sum, t0)

	.set	reorder				/* DADDI_WAR */

	.set	reorder				/* DADDI_WAR */

	ADD	dst, dst, NBYTES

	ADD	dst, dst, NBYTES
@@ -640,9 +662,9 @@ EXC( STORE t0, 0(dst), .Ls_exc) 
	li	t3, SHIFT_START # shift

	li	t3, SHIFT_START # shift

/* use .Ll_exc_copy here to return correct sum on fault */

/* use .Ll_exc_copy here to return correct sum on fault */

#define COPY_BYTE(N)			\

#define COPY_BYTE(N)			\

EXC(	lbu	t0, N(src), .Ll_exc_copy);	\

	LOADBU(t0, N(src), .Ll_exc_copy);	\

	SUB	len, len, 1;		\

	SUB	len, len, 1;		\

EXC(	sb	t0, N(dst), .Ls_exc);	\

	STOREB(t0, N(dst), .Ls_exc);	\

	SLLV	t0, t0, t3;		\

	SLLV	t0, t0, t3;		\

	addu	t3, SHIFT_INC;		\

	addu	t3, SHIFT_INC;		\

	beqz	len, .Lcopy_bytes_done; \

	beqz	len, .Lcopy_bytes_done; \
@@ -656,9 +678,9 @@ EXC( sb t0, N(dst), .Ls_exc); \ 
	COPY_BYTE(4)

	COPY_BYTE(4)

	COPY_BYTE(5)

	COPY_BYTE(5)

#endif

#endif

EXC(	lbu	t0, NBYTES-2(src), .Ll_exc_copy)

	LOADBU(t0, NBYTES-2(src), .Ll_exc_copy)

	SUB	len, len, 1

	SUB	len, len, 1

EXC(	sb	t0, NBYTES-2(dst), .Ls_exc)

	STOREB(t0, NBYTES-2(dst), .Ls_exc)

	SLLV	t0, t0, t3

	SLLV	t0, t0, t3

	or	t2, t0

	or	t2, t0

.Lcopy_bytes_done:

.Lcopy_bytes_done:
@@ -703,11 +725,11 @@ EXC( sb t0, NBYTES-2(dst), .Ls_exc) 
	 *

	 *

	 * Assumes src < THREAD_BUADDR($28)

	 * Assumes src < THREAD_BUADDR($28)

	 */

	 */

	LOAD	t0, TI_TASK($28)

	LOADK	t0, TI_TASK($28)

	 li	t2, SHIFT_START

	 li	t2, SHIFT_START

	LOAD	t0, THREAD_BUADDR(t0)

	LOADK	t0, THREAD_BUADDR(t0)

1:

1:

EXC(	lbu	t1, 0(src),	.Ll_exc)

	LOADBU(t1, 0(src), .Ll_exc)

	ADD	src, src, 1

	ADD	src, src, 1

	sb	t1, 0(dst)	# can't fault -- we're copy_from_user

	sb	t1, 0(dst)	# can't fault -- we're copy_from_user

	SLLV	t1, t1, t2

	SLLV	t1, t1, t2
@@ -718,9 +740,9 @@ EXC( lbu t1, 0(src), .Ll_exc) 
	bne	src, t0, 1b

	bne	src, t0, 1b

	.set	noreorder

	.set	noreorder

.Ll_exc:

.Ll_exc:

	LOAD	t0, TI_TASK($28)

	LOADK	t0, TI_TASK($28)

	 nop

	 nop

	LOAD	t0, THREAD_BUADDR(t0)	# t0 is just past last good address

	LOADK	t0, THREAD_BUADDR(t0)	# t0 is just past last good address

	 nop

	 nop

	SUB	len, AT, t0		# len number of uncopied bytes

	SUB	len, AT, t0		# len number of uncopied bytes

	/*

	/*