Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit aefa5688 authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Michael Ellerman
Browse files

powerpc/mm: don't do tlbie for updatepp request with NO HPTE fault 



upatepp can get called for a nohpte fault when we find from the linux
page table that the translation was hashed before. In that case
we are sure that there is no existing translation, hence we could
avoid doing tlbie.

We could possibly race with a parallel fault filling the TLB. But
that should be ok because updatepp is only ever relaxing permissions.
We also look at linux pte permission bits when filling hash pte
permission bits. We also hold the linux pte busy bits while
inserting/updating a hashpte entry, hence a paralle update of
linux pte is not possible. On the other hand mprotect involves
ptep_modify_prot_start which cause a hpte invalidate and not updatepp.

Performance number:
We use randbox_access_bench written by Anton.

Kernel with THP disabled and smaller hash page table size.

    86.60%  random_access_b  [kernel.kallsyms]                [k] .native_hpte_updatepp
     2.10%  random_access_b  random_access_bench              [.] doit
     1.99%  random_access_b  [kernel.kallsyms]                [k] .do_raw_spin_lock
     1.85%  random_access_b  [kernel.kallsyms]                [k] .native_hpte_insert
     1.26%  random_access_b  [kernel.kallsyms]                [k] .native_flush_hash_range
     1.18%  random_access_b  [kernel.kallsyms]                [k] .__delay
     0.69%  random_access_b  [kernel.kallsyms]                [k] .native_hpte_remove
     0.37%  random_access_b  [kernel.kallsyms]                [k] .clear_user_page
     0.34%  random_access_b  [kernel.kallsyms]                [k] .__hash_page_64K
     0.32%  random_access_b  [kernel.kallsyms]                [k] fast_exception_return
     0.30%  random_access_b  [kernel.kallsyms]                [k] .hash_page_mm

With Fix:

    27.54%  random_access_b  random_access_bench              [.] doit
    22.90%  random_access_b  [kernel.kallsyms]                [k] .native_hpte_insert
     5.76%  random_access_b  [kernel.kallsyms]                [k] .native_hpte_remove
     5.20%  random_access_b  [kernel.kallsyms]                [k] fast_exception_return
     5.12%  random_access_b  [kernel.kallsyms]                [k] .__hash_page_64K
     4.80%  random_access_b  [kernel.kallsyms]                [k] .hash_page_mm
     3.31%  random_access_b  [kernel.kallsyms]                [k] data_access_common
     1.84%  random_access_b  [kernel.kallsyms]                [k] .trace_hardirqs_on_caller

Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent abb90ee7

arch/powerpc/include/asm/machdep.h

+1 −1
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ struct machdep_calls { 
					 unsigned long newpp, 

					 unsigned long vpn,

					 int bpsize, int apsize,

					 int ssize, int local);

					 int ssize, unsigned long flags);

	void            (*hpte_updateboltedpp)(unsigned long newpp, 

					       unsigned long ea,

					       int psize, int ssize);

arch/powerpc/include/asm/mmu-hash64.h

+14 −8
Original line number Diff line number Diff line
@@ -316,27 +316,33 @@ static inline unsigned long hpt_hash(unsigned long vpn, 
	return hash & 0x7fffffffffUL;

}



#define HPTE_LOCAL_UPDATE	0x1

#define HPTE_NOHPTE_UPDATE	0x2



extern int __hash_page_4K(unsigned long ea, unsigned long access,

			  unsigned long vsid, pte_t *ptep, unsigned long trap,

			  unsigned int local, int ssize, int subpage_prot);

			  unsigned long flags, int ssize, int subpage_prot);

extern int __hash_page_64K(unsigned long ea, unsigned long access,

			   unsigned long vsid, pte_t *ptep, unsigned long trap,

			   unsigned int local, int ssize);

			   unsigned long flags, int ssize);

struct mm_struct;

unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap);

extern int hash_page_mm(struct mm_struct *mm, unsigned long ea, unsigned long access, unsigned long trap);

extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap);

extern int hash_page_mm(struct mm_struct *mm, unsigned long ea,

			unsigned long access, unsigned long trap,

			unsigned long flags);

extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap,

		     unsigned long dsisr);

int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,

		     pte_t *ptep, unsigned long trap, int local, int ssize,

		     unsigned int shift, unsigned int mmu_psize);

		     pte_t *ptep, unsigned long trap, unsigned long flags,

		     int ssize, unsigned int shift, unsigned int mmu_psize);

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

extern int __hash_page_thp(unsigned long ea, unsigned long access,

			   unsigned long vsid, pmd_t *pmdp, unsigned long trap,

			   int local, int ssize, unsigned int psize);

			   unsigned long flags, int ssize, unsigned int psize);

#else

static inline int __hash_page_thp(unsigned long ea, unsigned long access,

				  unsigned long vsid, pmd_t *pmdp,

				  unsigned long trap, int local,

				  unsigned long trap, unsigned long flags,

				  int ssize, unsigned int psize)

{

	BUG();

arch/powerpc/include/asm/tlbflush.h

+2 −2
Original line number Diff line number Diff line
@@ -125,11 +125,11 @@ static inline void arch_leave_lazy_mmu_mode(void) 




extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize,

			    int ssize, int local);

			    int ssize, unsigned long flags);

extern void flush_hash_range(unsigned long number, int local);

extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr,

				pmd_t *pmdp, unsigned int psize, int ssize,

				int local);

				unsigned long flags);



static inline void local_flush_tlb_mm(struct mm_struct *mm)

{

arch/powerpc/kernel/exceptions-64s.S

+2 −0
Original line number Diff line number Diff line
@@ -1565,9 +1565,11 @@ do_hash_page: 
	 * r3 contains the faulting address

	 * r4 contains the required access permissions

	 * r5 contains the trap number

	 * r6 contains dsisr

	 *

	 * at return r3 = 0 for success, 1 for page fault, negative for error

	 */

	ld      r6,_DSISR(r1)

	bl	hash_page		/* build HPTE if possible */

	cmpdi	r3,0			/* see if hash_page succeeded */

arch/powerpc/mm/hash_low_64.S

+8 −7
Original line number Diff line number Diff line
@@ -46,7 +46,8 @@ 


/*

 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,

 *		 pte_t *ptep, unsigned long trap, int local, int ssize)

 *		 pte_t *ptep, unsigned long trap, unsigned long flags,

 *		 int ssize)

 *

 * Adds a 4K page to the hash table in a segment of 4K pages only

 */
@@ -298,7 +299,7 @@ htab_modify_pte: 
	li	r6,MMU_PAGE_4K		/* base page size */

	li	r7,MMU_PAGE_4K		/* actual page size */

	ld	r8,STK_PARAM(R9)(r1)	/* segment size */

	ld	r9,STK_PARAM(R8)(r1)	/* get "local" param */

	ld	r9,STK_PARAM(R8)(r1)	/* get "flags" param */

.globl htab_call_hpte_updatepp

htab_call_hpte_updatepp:

	bl	.			/* Patched by htab_finish_init() */
@@ -338,8 +339,8 @@ htab_pte_insert_failure: 
 *****************************************************************************/



/* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,

 *		 pte_t *ptep, unsigned long trap, int local, int ssize,

 *		 int subpg_prot)

 *		 pte_t *ptep, unsigned long trap, unsigned local flags,

 *		 int ssize, int subpg_prot)

 */



/*
@@ -594,7 +595,7 @@ htab_inval_old_hpte: 
	li	r5,0			/* PTE.hidx */

	li	r6,MMU_PAGE_64K		/* psize */

	ld	r7,STK_PARAM(R9)(r1)	/* ssize */

	ld	r8,STK_PARAM(R8)(r1)	/* local */

	ld	r8,STK_PARAM(R8)(r1)	/* flags */

	bl	flush_hash_page

	/* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */

	lis	r0,_PAGE_HPTE_SUB@h
@@ -666,7 +667,7 @@ htab_modify_pte: 
	li	r6,MMU_PAGE_4K		/* base page size */

	li	r7,MMU_PAGE_4K		/* actual page size */

	ld	r8,STK_PARAM(R9)(r1)	/* segment size */

	ld	r9,STK_PARAM(R8)(r1)	/* get "local" param */

	ld	r9,STK_PARAM(R8)(r1)	/* get "flags" param */

.globl htab_call_hpte_updatepp

htab_call_hpte_updatepp:

	bl	.			/* patched by htab_finish_init() */
@@ -962,7 +963,7 @@ ht64_modify_pte: 
	li	r6,MMU_PAGE_64K		/* base page size */

	li	r7,MMU_PAGE_64K		/* actual page size */

	ld	r8,STK_PARAM(R9)(r1)	/* segment size */

	ld	r9,STK_PARAM(R8)(r1)	/* get "local" param */

	ld	r9,STK_PARAM(R8)(r1)	/* get "flags" param */

.globl ht64_call_hpte_updatepp

ht64_call_hpte_updatepp:

	bl	.			/* patched by htab_finish_init() */