Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 60e0a09d authored by Andrew Pinski's avatar Andrew Pinski Committed by Catalin Marinas
Browse files

arm64: lib: patch in prfm for copy_page if requested 

On ThunderX T88 pass 1 and pass 2, there is no hardware prefetching so
we need to patch in explicit software prefetching instructions

Prefetching improves this code by 60% over the original code and 2x
over the code without prefetching for the affected hardware using the
benchmark code at https://github.com/apinski-cavium/copy_page_benchmark



Signed-off-by: default avatarAndrew Pinski <apinski@cavium.com>
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
Tested-by: default avatarAndrew Pinski <apinski@cavium.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 223e23e8

arch/arm64/lib/copy_page.S

+17 −0
Original line number Diff line number Diff line
@@ -18,6 +18,8 @@ 
#include <linux/const.h>

#include <asm/assembler.h>

#include <asm/page.h>

#include <asm/cpufeature.h>

#include <asm/alternative.h>



/*

 * Copy a page from src to dest (both are page aligned)
@@ -27,6 +29,15 @@ 
 *	x1 - src

 */

ENTRY(copy_page)

alternative_if_not ARM64_HAS_NO_HW_PREFETCH

	nop

	nop

alternative_else

	# Prefetch two cache lines ahead.

	prfm    pldl1strm, [x1, #128]

	prfm    pldl1strm, [x1, #256]

alternative_endif



	ldp	x2, x3, [x1]

	ldp	x4, x5, [x1, #16]

	ldp	x6, x7, [x1, #32]
@@ -41,6 +52,12 @@ ENTRY(copy_page) 
1:

	subs	x18, x18, #128



alternative_if_not ARM64_HAS_NO_HW_PREFETCH

	nop

alternative_else

	prfm    pldl1strm, [x1, #384]

alternative_endif



	stnp	x2, x3, [x0]

	ldp	x2, x3, [x1]

	stnp	x4, x5, [x0, #16]