Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b3a04ed5 authored by Babu Moger's avatar Babu Moger Committed by David S. Miller
Browse files

arch/sparc: Optimized memcpy, memset, copy_to_user, copy_from_user for M7/M8 



New algorithm that takes advantage of the M7/M8 block init store
ASI, ie, overlapping pipelines and miss buffer filling.
Full details in code comments.

Signed-off-by: default avatarBabu Moger <babu.moger@oracle.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 1ab32693

arch/sparc/kernel/head_64.S

+14 −2
Original line number Diff line number Diff line
@@ -603,10 +603,10 @@ niagara_tlb_fixup: 
	be,pt	%xcc, niagara4_patch

	 nop

	cmp	%g1, SUN4V_CHIP_SPARC_M7

	be,pt	%xcc, niagara4_patch

	be,pt	%xcc, sparc_m7_patch

	 nop

	cmp	%g1, SUN4V_CHIP_SPARC_M8

	be,pt	%xcc, niagara4_patch

	be,pt	%xcc, sparc_m7_patch

	 nop

	cmp	%g1, SUN4V_CHIP_SPARC_SN

	be,pt	%xcc, niagara4_patch
@@ -621,6 +621,18 @@ niagara_tlb_fixup: 


	ba,a,pt	%xcc, 80f

	 nop



sparc_m7_patch:

	call	m7_patch_copyops

	 nop

	call	m7_patch_bzero

	 nop

	call	m7_patch_pageops

	 nop



	ba,a,pt	%xcc, 80f

	 nop



niagara4_patch:

	call	niagara4_patch_copyops

	 nop

arch/sparc/lib/M7copy_from_user.S

0 → 100644
+41 −0
Original line number Diff line number Diff line 
/*

 * M7copy_from_user.S: SPARC M7 optimized copy from userspace.

 *

 * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.

 */





#define EX_LD(x)			\

98:	x;				\

	.section __ex_table,"a";	\

	.align 4;			\

	.word 98b, __restore_asi;	\

	.text;				\

	.align 4;



#define EX_LD_FP(x)			\

98:	x;				\

	.section __ex_table,"a";	\

	.align 4;			\

	.word 98b, __restore_asi_fp;	\

	.text;				\

	.align 4;





#ifndef ASI_AIUS

#define ASI_AIUS	0x11

#endif



#define FUNC_NAME		M7copy_from_user

#define LOAD(type,addr,dest)	type##a [addr] %asi, dest

#define EX_RETVAL(x)		0



#ifdef __KERNEL__

#define PREAMBLE					\

	rd		%asi, %g1;			\

	cmp		%g1, ASI_AIUS;			\

	bne,pn		%icc, raw_copy_in_user;		\

	nop

#endif



#include "M7memcpy.S"

arch/sparc/lib/M7copy_to_user.S

0 → 100644
+51 −0
Original line number Diff line number Diff line 
/*

 * M7copy_to_user.S: SPARC M7 optimized copy to userspace.

 *

 * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.

 */





#define EX_ST(x)			\

98:	x;				\

	.section __ex_table,"a";	\

	.align 4;			\

	.word 98b, __restore_asi;	\

	.text;				\

	.align 4;



#define EX_ST_FP(x)			\

98:	x;				\

	.section __ex_table,"a";	\

	.align 4;			\

	.word 98b, __restore_asi_fp;	\

	.text;				\

	.align 4;





#ifndef ASI_AIUS

#define ASI_AIUS	0x11

#endif



#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS

#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23

#endif



#define FUNC_NAME		M7copy_to_user

#define STORE(type,src,addr)	type##a src, [addr] %asi

#define STORE_ASI		ASI_BLK_INIT_QUAD_LDD_AIUS

#define	STORE_MRU_ASI		ASI_ST_BLKINIT_MRU_S

#define EX_RETVAL(x)		0



#ifdef __KERNEL__

	/* Writing to %asi is _expensive_ so we hardcode it.

	 * Reading %asi to check for KERNEL_DS is comparatively

	 * cheap.

	 */

#define PREAMBLE					\

	rd		%asi, %g1;			\

	cmp		%g1, ASI_AIUS;			\

	bne,pn		%icc, raw_copy_in_user;		\

	nop

#endif



#include "M7memcpy.S"