Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 66c570f5 authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Michael Ellerman
Browse files

powerpc/mm: use _raw variant of page table accessors 



This switch few of the page table accessor to use the __raw variant
and does the cpu to big endian conversion of constants. This helps in
generating better code.

For ex: a pgd_none(pgd) check with and without fix is listed below

Without fix:
------------
   2240:	20 00 61 eb 	ld      r27,32(r1)
/* PGD level */
typedef struct { __be64 pgd; } pgd_t;
static inline unsigned long pgd_val(pgd_t x)
{
	return be64_to_cpu(x.pgd);

    2244:	22 00 66 78 	rldicl  r6,r3,32,32
    2248:	3e 40 7d 54 	rotlwi  r29,r3,8
    224c:	0e c0 7d 50 	rlwimi  r29,r3,24,0,7
    2250:	3e 40 c5 54 	rotlwi  r5,r6,8
    2254:	2e c4 7d 50 	rlwimi  r29,r3,24,16,23
    2258:	0e c0 c5 50 	rlwimi  r5,r6,24,0,7
    225c:	2e c4 c5 50 	rlwimi  r5,r6,24,16,23
    2260:	c6 07 bd 7b 	rldicr  r29,r29,32,31
    2264:	78 2b bd 7f 	or      r29,r29,r5
		if (pgd_none(pgd))
    2268:	00 00 bd 2f 	cmpdi   cr7,r29,0
    226c:	54 03 9e 41 	beq     cr7,25c0 <__get_user_pages_fast+0x500>

With fix:
---------
    2370:	20 00 61 eb 	ld      r27,32(r1)
		if (pgd_none(pgd))
    2374:	00 00 bd 2f 	cmpdi   cr7,r29,0
    2378:	a8 03 9e 41 	beq     cr7,2720 <__get_user_pages_fast+0x530>
			break;
Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: default avatarBalbir Singh <bsingharora@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent bf16cdf4

arch/powerpc/include/asm/book3s/64/pgtable-4k.h

+3 −3
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ static inline int pmd_huge(pmd_t pmd) 
	 * leaf pte for huge page

	 */

	if (radix_enabled())

		return !!(pmd_val(pmd) & _PAGE_PTE);

		return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));

	return 0;

}
@@ -21,7 +21,7 @@ static inline int pud_huge(pud_t pud) 
	 * leaf pte for huge page

	 */

	if (radix_enabled())

		return !!(pud_val(pud) & _PAGE_PTE);

		return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE));

	return 0;

}
@@ -31,7 +31,7 @@ static inline int pgd_huge(pgd_t pgd) 
	 * leaf pte for huge page

	 */

	if (radix_enabled())

		return !!(pgd_val(pgd) & _PAGE_PTE);

		return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE));

	return 0;

}

#define pgd_huge pgd_huge

arch/powerpc/include/asm/book3s/64/pgtable-64k.h

+3 −3
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ static inline int pmd_huge(pmd_t pmd) 
	/*

	 * leaf pte for huge page

	 */

	return !!(pmd_val(pmd) & _PAGE_PTE);

	return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));

}



static inline int pud_huge(pud_t pud)
@@ -23,7 +23,7 @@ static inline int pud_huge(pud_t pud) 
	/*

	 * leaf pte for huge page

	 */

	return !!(pud_val(pud) & _PAGE_PTE);

	return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE));

}



static inline int pgd_huge(pgd_t pgd)
@@ -31,7 +31,7 @@ static inline int pgd_huge(pgd_t pgd) 
	/*

	 * leaf pte for huge page

	 */

	return !!(pgd_val(pgd) & _PAGE_PTE);

	return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE));

}

#define pgd_huge pgd_huge

arch/powerpc/include/asm/book3s/64/pgtable.h

+70 −29
Original line number Diff line number Diff line
@@ -318,7 +318,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, 
{

	unsigned long old;



	if ((pte_val(*ptep) & (_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)

	if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)

		return 0;

	old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);

	return (old & _PAGE_ACCESSED) != 0;
@@ -336,8 +336,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, 
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,

				      pte_t *ptep)

{



	if ((pte_val(*ptep) & _PAGE_WRITE) == 0)

	if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_WRITE)) == 0)

		return;



	pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0);
@@ -346,7 +345,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, 
static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,

					   unsigned long addr, pte_t *ptep)

{

	if ((pte_val(*ptep) & _PAGE_WRITE) == 0)

	if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_WRITE)) == 0)

		return;



	pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1);
@@ -365,17 +364,35 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, 
{

	pte_update(mm, addr, ptep, ~0UL, 0, 0);

}

static inline int pte_write(pte_t pte)		{ return !!(pte_val(pte) & _PAGE_WRITE);}

static inline int pte_dirty(pte_t pte)		{ return !!(pte_val(pte) & _PAGE_DIRTY); }

static inline int pte_young(pte_t pte)		{ return !!(pte_val(pte) & _PAGE_ACCESSED); }

static inline int pte_special(pte_t pte)	{ return !!(pte_val(pte) & _PAGE_SPECIAL); }



static inline int pte_write(pte_t pte)

{

	return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE));

}



static inline int pte_dirty(pte_t pte)

{

	return !!(pte_raw(pte) & cpu_to_be64(_PAGE_DIRTY));

}



static inline int pte_young(pte_t pte)

{

	return !!(pte_raw(pte) & cpu_to_be64(_PAGE_ACCESSED));

}



static inline int pte_special(pte_t pte)

{

	return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SPECIAL));

}



static inline pgprot_t pte_pgprot(pte_t pte)	{ return __pgprot(pte_val(pte) & PAGE_PROT_BITS); }



#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY

static inline bool pte_soft_dirty(pte_t pte)

{

	return !!(pte_val(pte) & _PAGE_SOFT_DIRTY);

	return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SOFT_DIRTY));

}



static inline pte_t pte_mksoft_dirty(pte_t pte)

{

	return __pte(pte_val(pte) | _PAGE_SOFT_DIRTY);
@@ -395,14 +412,14 @@ static inline pte_t pte_clear_soft_dirty(pte_t pte) 
 */

static inline int pte_protnone(pte_t pte)

{

	return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PRIVILEGED)) ==

		(_PAGE_PRESENT | _PAGE_PRIVILEGED);

	return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED)) ==

		cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED);

}

#endif /* CONFIG_NUMA_BALANCING */



static inline int pte_present(pte_t pte)

{

	return !!(pte_val(pte) & _PAGE_PRESENT);

	return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT));

}

/*

 * Conversion functions: convert a page and protection to a page entry,
@@ -474,7 +491,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) 


static inline bool pte_user(pte_t pte)

{

	return !(pte_val(pte) & _PAGE_PRIVILEGED);

	return !(pte_raw(pte) & cpu_to_be64(_PAGE_PRIVILEGED));

}



/* Encode and de-code a swap entry */
@@ -517,10 +534,12 @@ static inline pte_t pte_swp_mksoft_dirty(pte_t pte) 
{

	return __pte(pte_val(pte) | _PAGE_SWP_SOFT_DIRTY);

}



static inline bool pte_swp_soft_dirty(pte_t pte)

{

	return !!(pte_val(pte) & _PAGE_SWP_SOFT_DIRTY);

	return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_SOFT_DIRTY));

}



static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)

{

	return __pte(pte_val(pte) & ~_PAGE_SWP_SOFT_DIRTY);
@@ -626,8 +645,16 @@ static inline void pmd_clear(pmd_t *pmdp) 
	*pmdp = __pmd(0);

}



#define pmd_none(pmd)		(!pmd_val(pmd))

#define	pmd_present(pmd)	(!pmd_none(pmd))

static inline int pmd_none(pmd_t pmd)

{

	return !pmd_raw(pmd);

}



static inline int pmd_present(pmd_t pmd)

{



	return !pmd_none(pmd);

}



static inline int pmd_bad(pmd_t pmd)

{
@@ -646,19 +673,26 @@ static inline void pud_clear(pud_t *pudp) 
	*pudp = __pud(0);

}



#define pud_none(pud)		(!pud_val(pud))

#define pud_present(pud)	(pud_val(pud) != 0)

static inline int pud_none(pud_t pud)

{

	return !pud_raw(pud);

}



static inline int pud_present(pud_t pud)

{

	return !pud_none(pud);

}



extern struct page *pud_page(pud_t pud);

extern struct page *pmd_page(pmd_t pmd);

static inline pte_t pud_pte(pud_t pud)

{

	return __pte(pud_val(pud));

	return __pte_raw(pud_raw(pud));

}



static inline pud_t pte_pud(pte_t pte)

{

	return __pud(pte_val(pte));

	return __pud_raw(pte_raw(pte));

}

#define pud_write(pud)		pte_write(pud_pte(pud))
@@ -681,17 +715,24 @@ static inline void pgd_clear(pgd_t *pgdp) 
	*pgdp = __pgd(0);

}



#define pgd_none(pgd)		(!pgd_val(pgd))

#define pgd_present(pgd)	(!pgd_none(pgd))

static inline int pgd_none(pgd_t pgd)

{

	return !pgd_raw(pgd);

}



static inline int pgd_present(pgd_t pgd)

{

	return !pgd_none(pgd);

}



static inline pte_t pgd_pte(pgd_t pgd)

{

	return __pte(pgd_val(pgd));

	return __pte_raw(pgd_raw(pgd));

}



static inline pgd_t pte_pgd(pte_t pte)

{

	return __pgd(pte_val(pte));

	return __pgd_raw(pte_raw(pte));

}



static inline int pgd_bad(pgd_t pgd)
@@ -783,12 +824,12 @@ struct page *realmode_pfn_to_page(unsigned long pfn); 


static inline pte_t pmd_pte(pmd_t pmd)

{

	return __pte(pmd_val(pmd));

	return __pte_raw(pmd_raw(pmd));

}



static inline pmd_t pte_pmd(pte_t pte)

{

	return __pmd(pte_val(pte));

	return __pmd_raw(pte_raw(pte));

}



static inline pte_t *pmdp_ptep(pmd_t *pmd)
@@ -849,7 +890,7 @@ pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, 


static inline int pmd_large(pmd_t pmd)

{

	return !!(pmd_val(pmd) & _PAGE_PTE);

	return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));

}



static inline pmd_t pmd_mknotpresent(pmd_t pmd)
@@ -865,7 +906,7 @@ static inline int __pmdp_test_and_clear_young(struct mm_struct *mm, 
{

	unsigned long old;



	if ((pmd_val(*pmdp) & (_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)

	if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)

		return 0;

	old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0);

	return ((old & _PAGE_ACCESSED) != 0);
@@ -876,7 +917,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, 
				      pmd_t *pmdp)

{



	if ((pmd_val(*pmdp) & _PAGE_WRITE) == 0)

	if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_WRITE)) == 0)

		return;



	pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0);

arch/powerpc/include/asm/pgtable-be-types.h

+15 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ 
/* PTE level */

typedef struct { __be64 pte; } pte_t;

#define __pte(x)	((pte_t) { cpu_to_be64(x) })

#define __pte_raw(x)	((pte_t) { (x) })

static inline unsigned long pte_val(pte_t x)

{

	return be64_to_cpu(x.pte);
@@ -20,6 +21,7 @@ static inline __be64 pte_raw(pte_t x) 
#ifdef CONFIG_PPC64

typedef struct { __be64 pmd; } pmd_t;

#define __pmd(x)	((pmd_t) { cpu_to_be64(x) })

#define __pmd_raw(x)	((pmd_t) { (x) })

static inline unsigned long pmd_val(pmd_t x)

{

	return be64_to_cpu(x.pmd);
@@ -37,21 +39,34 @@ static inline __be64 pmd_raw(pmd_t x) 
#if defined(CONFIG_PPC_BOOK3S_64) || !defined(CONFIG_PPC_64K_PAGES)

typedef struct { __be64 pud; } pud_t;

#define __pud(x)	((pud_t) { cpu_to_be64(x) })

#define __pud_raw(x)	((pud_t) { (x) })

static inline unsigned long pud_val(pud_t x)

{

	return be64_to_cpu(x.pud);

}



static inline __be64 pud_raw(pud_t x)

{

	return x.pud;

}



#endif /* CONFIG_PPC_BOOK3S_64 || !CONFIG_PPC_64K_PAGES */

#endif /* CONFIG_PPC64 */



/* PGD level */

typedef struct { __be64 pgd; } pgd_t;

#define __pgd(x)	((pgd_t) { cpu_to_be64(x) })

#define __pgd_raw(x)	((pgd_t) { (x) })

static inline unsigned long pgd_val(pgd_t x)

{

	return be64_to_cpu(x.pgd);

}



static inline __be64 pgd_raw(pgd_t x)

{

	return x.pgd;

}



/* Page protection bits */

typedef struct { unsigned long pgprot; } pgprot_t;

#define pgprot_val(x)	((x).pgprot)