Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f2a6a705 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Ingo Molnar
Browse files

x86: Convert the rest of the code to support p4d_t 



This patch converts x86 to use proper folding of a new (fifth) page table level
with <asm-generic/pgtable-nop4d.h>.

That's a bit of a kitchen sink patch, but I don't see how to split it further
without hurting bisectability.

Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: linux-arch@vger.kernel.org
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20170317185515.8636-7-kirill.shutemov@linux.intel.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 907cd439

arch/x86/include/asm/paravirt.h

+28 −5
Original line number Diff line number Diff line
@@ -536,7 +536,7 @@ static inline void set_pud(pud_t *pudp, pud_t pud) 
		PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,

			    val);

}

#if CONFIG_PGTABLE_LEVELS == 4

#if CONFIG_PGTABLE_LEVELS >= 4

static inline pud_t __pud(pudval_t val)

{

	pudval_t ret;
@@ -565,6 +565,32 @@ static inline pudval_t pud_val(pud_t pud) 
	return ret;

}



static inline void pud_clear(pud_t *pudp)

{

	set_pud(pudp, __pud(0));

}



static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)

{

	p4dval_t val = native_p4d_val(p4d);



	if (sizeof(p4dval_t) > sizeof(long))

		PVOP_VCALL3(pv_mmu_ops.set_p4d, p4dp,

			    val, (u64)val >> 32);

	else

		PVOP_VCALL2(pv_mmu_ops.set_p4d, p4dp,

			    val);

}



static inline void p4d_clear(p4d_t *p4dp)

{

	set_p4d(p4dp, __p4d(0));

}



#if CONFIG_PGTABLE_LEVELS >= 5



#error FIXME



static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)

{

	pgdval_t val = native_pgd_val(pgd);
@@ -582,10 +608,7 @@ static inline void pgd_clear(pgd_t *pgdp) 
	set_pgd(pgdp, __pgd(0));

}



static inline void pud_clear(pud_t *pudp)

{

	set_pud(pudp, __pud(0));

}

#endif  /* CONFIG_PGTABLE_LEVELS == 5 */



#endif	/* CONFIG_PGTABLE_LEVELS == 4 */

arch/x86/include/asm/paravirt_types.h

+9 −3
Original line number Diff line number Diff line
@@ -279,12 +279,18 @@ struct pv_mmu_ops { 
	struct paravirt_callee_save pmd_val;

	struct paravirt_callee_save make_pmd;



#if CONFIG_PGTABLE_LEVELS == 4

#if CONFIG_PGTABLE_LEVELS >= 4

	struct paravirt_callee_save pud_val;

	struct paravirt_callee_save make_pud;



	void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);

#endif	/* CONFIG_PGTABLE_LEVELS == 4 */

	void (*set_p4d)(p4d_t *p4dp, p4d_t p4dval);



#if CONFIG_PGTABLE_LEVELS >= 5

#error FIXME

#endif	/* CONFIG_PGTABLE_LEVELS >= 5 */



#endif	/* CONFIG_PGTABLE_LEVELS >= 4 */



#endif	/* CONFIG_PGTABLE_LEVELS >= 3 */



	struct pv_lazy_ops lazy_mode;

arch/x86/include/asm/pgalloc.h

+33 −2
Original line number Diff line number Diff line
@@ -121,10 +121,10 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 
#endif	/* CONFIG_X86_PAE */



#if CONFIG_PGTABLE_LEVELS > 3

static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)

static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)

{

	paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);

	set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud)));

	set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud)));

}



static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
@@ -150,6 +150,37 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, 
	___pud_free_tlb(tlb, pud);

}



#if CONFIG_PGTABLE_LEVELS > 4

static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)

{

	paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);

	set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));

}



static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)

{

	gfp_t gfp = GFP_KERNEL_ACCOUNT;



	if (mm == &init_mm)

		gfp &= ~__GFP_ACCOUNT;

	return (p4d_t *)get_zeroed_page(gfp);

}



static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)

{

	BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));

	free_page((unsigned long)p4d);

}



extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);



static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,

				  unsigned long address)

{

	___p4d_free_tlb(tlb, p4d);

}



#endif	/* CONFIG_PGTABLE_LEVELS > 4 */

#endif	/* CONFIG_PGTABLE_LEVELS > 3 */

#endif	/* CONFIG_PGTABLE_LEVELS > 2 */

arch/x86/include/asm/pgtable.h

+54 −5
Original line number Diff line number Diff line
@@ -53,11 +53,19 @@ extern struct mm_struct *pgd_page_get_mm(struct page *page); 


#define set_pmd(pmdp, pmd)		native_set_pmd(pmdp, pmd)



#ifndef __PAGETABLE_PUD_FOLDED

#ifndef __PAGETABLE_P4D_FOLDED

#define set_pgd(pgdp, pgd)		native_set_pgd(pgdp, pgd)

#define pgd_clear(pgd)			native_pgd_clear(pgd)

#endif



#ifndef set_p4d

# define set_p4d(p4dp, p4d)		native_set_p4d(p4dp, p4d)

#endif



#ifndef __PAGETABLE_PUD_FOLDED

#define p4d_clear(p4d)			native_p4d_clear(p4d)

#endif



#ifndef set_pud

# define set_pud(pudp, pud)		native_set_pud(pudp, pud)

#endif
@@ -74,6 +82,11 @@ extern struct mm_struct *pgd_page_get_mm(struct page *page); 
#define pgd_val(x)	native_pgd_val(x)

#define __pgd(x)	native_make_pgd(x)



#ifndef __PAGETABLE_P4D_FOLDED

#define p4d_val(x)	native_p4d_val(x)

#define __p4d(x)	native_make_p4d(x)

#endif



#ifndef __PAGETABLE_PUD_FOLDED

#define pud_val(x)	native_pud_val(x)

#define __pud(x)	native_make_pud(x)
@@ -554,6 +567,7 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 
#define pte_pgprot(x) __pgprot(pte_flags(x))

#define pmd_pgprot(x) __pgprot(pmd_flags(x))

#define pud_pgprot(x) __pgprot(pud_flags(x))

#define p4d_pgprot(x) __pgprot(p4d_flags(x))



#define canon_pgprot(p) __pgprot(massage_pgprot(p))
@@ -792,12 +806,47 @@ static inline unsigned long pud_index(unsigned long address) 
	return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);

}



#if CONFIG_PGTABLE_LEVELS > 3

static inline int p4d_none(p4d_t p4d)

{

	return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;

}



static inline int p4d_present(p4d_t p4d)

{

	return p4d_flags(p4d) & _PAGE_PRESENT;

}



static inline unsigned long p4d_page_vaddr(p4d_t p4d)

{

	return (unsigned long)__va(p4d_val(p4d) & p4d_pfn_mask(p4d));

}



/*

 * Currently stuck as a macro due to indirect forward reference to

 * linux/mmzone.h's __section_mem_map_addr() definition:

 */

#define p4d_page(p4d)		\

	pfn_to_page((p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT)



/* Find an entry in the third-level page table.. */

static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)

{

	return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address);

}



static inline int p4d_bad(p4d_t p4d)

{

	return (p4d_flags(p4d) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;

}

#endif  /* CONFIG_PGTABLE_LEVELS > 3 */



static inline unsigned long p4d_index(unsigned long address)

{

	return (address >> P4D_SHIFT) & (PTRS_PER_P4D - 1);

}



#if CONFIG_PGTABLE_LEVELS > 3

#if CONFIG_PGTABLE_LEVELS > 4

static inline int pgd_present(pgd_t pgd)

{

	return pgd_flags(pgd) & _PAGE_PRESENT;
@@ -815,9 +864,9 @@ static inline unsigned long pgd_page_vaddr(pgd_t pgd) 
#define pgd_page(pgd)		pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)



/* to find an entry in a page-table-directory. */

static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)

static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)

{

	return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(address);

	return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address);

}



static inline int pgd_bad(pgd_t pgd)
@@ -835,7 +884,7 @@ static inline int pgd_none(pgd_t pgd) 
	 */

	return !native_pgd_val(pgd);

}

#endif	/* CONFIG_PGTABLE_LEVELS > 3 */

#endif	/* CONFIG_PGTABLE_LEVELS > 4 */



#endif	/* __ASSEMBLY__ */

arch/x86/include/asm/pgtable_64.h

+11 −1
Original line number Diff line number Diff line
@@ -41,9 +41,9 @@ extern void paging_init(void); 


struct mm_struct;



void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte);

void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte);





static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,

				    pte_t *ptep)

{
@@ -121,6 +121,16 @@ static inline pud_t native_pudp_get_and_clear(pud_t *xp) 
#endif

}



static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)

{

	*p4dp = p4d;

}



static inline void native_p4d_clear(p4d_t *p4d)

{

	native_set_p4d(p4d, (p4d_t) { .pgd = native_make_pgd(0)});

}



static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)

{

	*pgdp = pgd;