Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ae2c6ca6 authored by David S. Miller's avatar David S. Miller
Browse files

sparc64: Add SPARC-T4 optimized memcpy. 



		Before		After
		--------------	--------------
bw_tcp:         1288.53 MB/sec	1637.77 MB/sec
bw_pipe:        1517.18 MB/sec	2107.61 MB/sec
bw_unix:        1838.38 MB/sec	2640.91 MB/sec

make -s -j128
allmodconfig	5min 49sec	5min 31sec

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent da201161

arch/sparc/kernel/head_64.S

+12 −2
Original line number Diff line number Diff line
@@ -559,10 +559,10 @@ niagara_tlb_fixup: 
	be,pt	%xcc, niagara2_patch

	 nop

	cmp	%g1, SUN4V_CHIP_NIAGARA4

	be,pt	%xcc, niagara2_patch

	be,pt	%xcc, niagara4_patch

	 nop

	cmp	%g1, SUN4V_CHIP_NIAGARA5

	be,pt	%xcc, niagara2_patch

	be,pt	%xcc, niagara4_patch

	 nop



	call	generic_patch_copyops
@@ -573,6 +573,16 @@ niagara_tlb_fixup: 
	 nop



	ba,a,pt	%xcc, 80f

niagara4_patch:

	call	niagara4_patch_copyops

	 nop

	call	niagara_patch_bzero

	 nop

	call	niagara4_patch_pageops

	 nop



	ba,a,pt	%xcc, 80f



niagara2_patch:

	call	niagara2_patch_copyops

	 nop

arch/sparc/lib/Makefile

+3 −0
Original line number Diff line number Diff line
@@ -32,6 +32,9 @@ lib-$(CONFIG_SPARC64) += NGpatch.o NGpage.o NGbzero.o 
lib-$(CONFIG_SPARC64) += NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o

lib-$(CONFIG_SPARC64) +=  NG2patch.o



lib-$(CONFIG_SPARC64) += NG4memcpy.o NG4copy_from_user.o NG4copy_to_user.o

lib-$(CONFIG_SPARC64) +=  NG4patch.o NG4copy_page.o



lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o

lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o

arch/sparc/lib/NG4copy_from_user.S

0 → 100644
+30 −0
Original line number Diff line number Diff line 
/* NG4copy_from_user.S: Niagara-2 optimized copy from userspace.

 *

 * Copyright (C) 2012 David S. Miller (davem@davemloft.net)

 */



#define EX_LD(x)		\

98:	x;			\

	.section __ex_table,"a";\

	.align 4;		\

	.word 98b, __retl_one_asi;\

	.text;			\

	.align 4;



#ifndef ASI_AIUS

#define ASI_AIUS	0x11

#endif



#define FUNC_NAME		NG4copy_from_user

#define LOAD(type,addr,dest)	type##a [addr] %asi, dest

#define EX_RETVAL(x)		0



#ifdef __KERNEL__

#define PREAMBLE					\

	rd		%asi, %g1;			\

	cmp		%g1, ASI_AIUS;			\

	bne,pn		%icc, ___copy_in_user;		\

	 nop

#endif



#include "NG4memcpy.S"

arch/sparc/lib/NG4copy_page.S

0 → 100644
+57 −0
Original line number Diff line number Diff line 
/* NG4copy_page.S: Niagara-4 optimized copy page.

 *

 * Copyright (C) 2012 (davem@davemloft.net)

 */



#include <asm/asi.h>

#include <asm/page.h>



	.text

	.align		32



	.register	%g2, #scratch

	.register	%g3, #scratch



	.globl		NG4copy_user_page

NG4copy_user_page:	/* %o0=dest, %o1=src, %o2=vaddr */

	prefetch	[%o1 + 0x000], #n_reads_strong

	prefetch	[%o1 + 0x040], #n_reads_strong

	prefetch	[%o1 + 0x080], #n_reads_strong

	prefetch	[%o1 + 0x0c0], #n_reads_strong

	set		PAGE_SIZE, %g7

	prefetch	[%o1 + 0x100], #n_reads_strong

	prefetch	[%o1 + 0x140], #n_reads_strong

	prefetch	[%o1 + 0x180], #n_reads_strong

	prefetch	[%o1 + 0x1c0], #n_reads_strong

1:

	ldx		[%o1 + 0x00], %o2

	subcc		%g7, 0x40, %g7

	ldx		[%o1 + 0x08], %o3

	ldx		[%o1 + 0x10], %o4

	ldx		[%o1 + 0x18], %o5

	ldx		[%o1 + 0x20], %g1

	stxa		%o2, [%o0] ASI_BLK_INIT_QUAD_LDD_P

	add		%o0, 0x08, %o0

	ldx		[%o1 + 0x28], %g2

	stxa		%o3, [%o0] ASI_BLK_INIT_QUAD_LDD_P

	add		%o0, 0x08, %o0

	ldx		[%o1 + 0x30], %g3

	stxa		%o4, [%o0] ASI_BLK_INIT_QUAD_LDD_P

	add		%o0, 0x08, %o0

	ldx		[%o1 + 0x38], %o2

	add		%o1, 0x40, %o1

	stxa		%o5, [%o0] ASI_BLK_INIT_QUAD_LDD_P

	add		%o0, 0x08, %o0

	stxa		%g1, [%o0] ASI_BLK_INIT_QUAD_LDD_P

	add		%o0, 0x08, %o0

	stxa		%g2, [%o0] ASI_BLK_INIT_QUAD_LDD_P

	add		%o0, 0x08, %o0

	stxa		%g3, [%o0] ASI_BLK_INIT_QUAD_LDD_P

	add		%o0, 0x08, %o0

	stxa		%o2, [%o0] ASI_BLK_INIT_QUAD_LDD_P

	add		%o0, 0x08, %o0

	bne,pt		%icc, 1b

	 prefetch	[%o1 + 0x200], #n_reads_strong

	retl

	 membar		#StoreLoad | #StoreStore

	.size		NG4copy_user_page,.-NG4copy_user_page

arch/sparc/lib/NG4copy_to_user.S

0 → 100644
+39 −0
Original line number Diff line number Diff line 
/* NG4copy_to_user.S: Niagara-4 optimized copy to userspace.

 *

 * Copyright (C) 2012 David S. Miller (davem@davemloft.net)

 */



#define EX_ST(x)		\

98:	x;			\

	.section __ex_table,"a";\

	.align 4;		\

	.word 98b, __retl_one_asi;\

	.text;			\

	.align 4;



#ifndef ASI_AIUS

#define ASI_AIUS	0x11

#endif



#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS

#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23

#endif



#define FUNC_NAME		NG4copy_to_user

#define STORE(type,src,addr)	type##a src, [addr] %asi

#define STORE_ASI		ASI_BLK_INIT_QUAD_LDD_AIUS

#define EX_RETVAL(x)		0



#ifdef __KERNEL__

	/* Writing to %asi is _expensive_ so we hardcode it.

	 * Reading %asi to check for KERNEL_DS is comparatively

	 * cheap.

	 */

#define PREAMBLE					\

	rd		%asi, %g1;			\

	cmp		%g1, ASI_AIUS;			\

	bne,pn		%icc, ___copy_in_user;		\

	 nop

#endif



#include "NG4memcpy.S"