Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e0c4f675 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Ingo Molnar
Browse files

x86/mm: Convert trivial cases of page table walk to 5-level paging 



This patch only covers simple cases. Less trivial cases will be
converted with separate patches.

Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-arch@vger.kernel.org
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20170313143309.16020-3-kirill.shutemov@linux.intel.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent fe1e8c3e

arch/x86/kernel/tboot.c

+5 −1
Original line number Original line Diff line number Diff line
@@ -118,12 +118,16 @@ static int map_tboot_page(unsigned long vaddr, unsigned long pfn, 
			  pgprot_t prot)

			  pgprot_t prot)

{

{

	pgd_t *pgd;

	pgd_t *pgd;

	p4d_t *p4d;

	pud_t *pud;

	pud_t *pud;

	pmd_t *pmd;

	pmd_t *pmd;

	pte_t *pte;

	pte_t *pte;





	pgd = pgd_offset(&tboot_mm, vaddr);

	pgd = pgd_offset(&tboot_mm, vaddr);

	pud = pud_alloc(&tboot_mm, pgd, vaddr);

	p4d = p4d_alloc(&tboot_mm, pgd, vaddr);

	if (!p4d)

		return -1;

	pud = pud_alloc(&tboot_mm, p4d, vaddr);

	if (!pud)

	if (!pud)

		return -1;

		return -1;

	pmd = pmd_alloc(&tboot_mm, pud, vaddr);

	pmd = pmd_alloc(&tboot_mm, pud, vaddr);

arch/x86/kernel/vm86_32.c

+5 −1
Original line number Original line Diff line number Diff line
@@ -164,6 +164,7 @@ static void mark_screen_rdonly(struct mm_struct *mm) 
	struct vm_area_struct *vma;

	struct vm_area_struct *vma;

	spinlock_t *ptl;

	spinlock_t *ptl;

	pgd_t *pgd;

	pgd_t *pgd;

	p4d_t *p4d;

	pud_t *pud;

	pud_t *pud;

	pmd_t *pmd;

	pmd_t *pmd;

	pte_t *pte;

	pte_t *pte;
@@ -173,7 +174,10 @@ static void mark_screen_rdonly(struct mm_struct *mm) 
	pgd = pgd_offset(mm, 0xA0000);

	pgd = pgd_offset(mm, 0xA0000);

	if (pgd_none_or_clear_bad(pgd))

	if (pgd_none_or_clear_bad(pgd))

		goto out;

		goto out;

	pud = pud_offset(pgd, 0xA0000);

	p4d = p4d_offset(pgd, 0xA0000);

	if (p4d_none_or_clear_bad(p4d))

		goto out;

	pud = pud_offset(p4d, 0xA0000);

	if (pud_none_or_clear_bad(pud))

	if (pud_none_or_clear_bad(pud))

		goto out;

		goto out;

	pmd = pmd_offset(pud, 0xA0000);

	pmd = pmd_offset(pud, 0xA0000);

arch/x86/mm/fault.c

+33 −6
Original line number Original line Diff line number Diff line
@@ -253,6 +253,7 @@ static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) 
{

{

	unsigned index = pgd_index(address);

	unsigned index = pgd_index(address);

	pgd_t *pgd_k;

	pgd_t *pgd_k;

	p4d_t *p4d, *p4d_k;

	pud_t *pud, *pud_k;

	pud_t *pud, *pud_k;

	pmd_t *pmd, *pmd_k;

	pmd_t *pmd, *pmd_k;
@@ -265,10 +266,15 @@ static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address) 
	/*

	/*

	 * set_pgd(pgd, *pgd_k); here would be useless on PAE

	 * set_pgd(pgd, *pgd_k); here would be useless on PAE

	 * and redundant with the set_pmd() on non-PAE. As would

	 * and redundant with the set_pmd() on non-PAE. As would

	 * set_pud.

	 * set_p4d/set_pud.

	 */

	 */

	pud = pud_offset(pgd, address);

	p4d = p4d_offset(pgd, address);

	pud_k = pud_offset(pgd_k, address);

	p4d_k = p4d_offset(pgd_k, address);

	if (!p4d_present(*p4d_k))

		return NULL;



	pud = pud_offset(p4d, address);

	pud_k = pud_offset(p4d_k, address);

	if (!pud_present(*pud_k))

	if (!pud_present(*pud_k))

		return NULL;

		return NULL;
@@ -384,6 +390,8 @@ static void dump_pagetable(unsigned long address) 
{

{

	pgd_t *base = __va(read_cr3());

	pgd_t *base = __va(read_cr3());

	pgd_t *pgd = &base[pgd_index(address)];

	pgd_t *pgd = &base[pgd_index(address)];

	p4d_t *p4d;

	pud_t *pud;

	pmd_t *pmd;

	pmd_t *pmd;

	pte_t *pte;

	pte_t *pte;
@@ -392,7 +400,9 @@ static void dump_pagetable(unsigned long address) 
	if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))

	if (!low_pfn(pgd_val(*pgd) >> PAGE_SHIFT) || !pgd_present(*pgd))

		goto out;

		goto out;

#endif

#endif

	pmd = pmd_offset(pud_offset(pgd, address), address);

	p4d = p4d_offset(pgd, address);

	pud = pud_offset(p4d, address);

	pmd = pmd_offset(pud, address);

	printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));

	printk(KERN_CONT "*pde = %0*Lx ", sizeof(*pmd) * 2, (u64)pmd_val(*pmd));





	/*

	/*
@@ -526,6 +536,7 @@ static void dump_pagetable(unsigned long address) 
{

{

	pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK);

	pgd_t *base = __va(read_cr3() & PHYSICAL_PAGE_MASK);

	pgd_t *pgd = base + pgd_index(address);

	pgd_t *pgd = base + pgd_index(address);

	p4d_t *p4d;

	pud_t *pud;

	pud_t *pud;

	pmd_t *pmd;

	pmd_t *pmd;

	pte_t *pte;

	pte_t *pte;
@@ -538,7 +549,15 @@ static void dump_pagetable(unsigned long address) 
	if (!pgd_present(*pgd))

	if (!pgd_present(*pgd))

		goto out;

		goto out;





	pud = pud_offset(pgd, address);

	p4d = p4d_offset(pgd, address);

	if (bad_address(p4d))

		goto bad;



	printk("P4D %lx ", p4d_val(*p4d));

	if (!p4d_present(*p4d) || p4d_large(*p4d))

		goto out;



	pud = pud_offset(p4d, address);

	if (bad_address(pud))

	if (bad_address(pud))

		goto bad;

		goto bad;
@@ -1082,6 +1101,7 @@ static noinline int 
spurious_fault(unsigned long error_code, unsigned long address)

spurious_fault(unsigned long error_code, unsigned long address)

{

{

	pgd_t *pgd;

	pgd_t *pgd;

	p4d_t *p4d;

	pud_t *pud;

	pud_t *pud;

	pmd_t *pmd;

	pmd_t *pmd;

	pte_t *pte;

	pte_t *pte;
@@ -1104,7 +1124,14 @@ spurious_fault(unsigned long error_code, unsigned long address) 
	if (!pgd_present(*pgd))

	if (!pgd_present(*pgd))

		return 0;

		return 0;





	pud = pud_offset(pgd, address);

	p4d = p4d_offset(pgd, address);

	if (!p4d_present(*p4d))

		return 0;



	if (p4d_large(*p4d))

		return spurious_fault_check(error_code, (pte_t *) p4d);



	pud = pud_offset(p4d, address);

	if (!pud_present(*pud))

	if (!pud_present(*pud))

		return 0;

		return 0;

arch/x86/mm/init_32.c

+16 −6
Original line number Original line Diff line number Diff line
@@ -67,6 +67,7 @@ bool __read_mostly __vmalloc_start_set = false; 
 */

 */

static pmd_t * __init one_md_table_init(pgd_t *pgd)

static pmd_t * __init one_md_table_init(pgd_t *pgd)

{

{

	p4d_t *p4d;

	pud_t *pud;

	pud_t *pud;

	pmd_t *pmd_table;

	pmd_t *pmd_table;
@@ -75,13 +76,15 @@ static pmd_t * __init one_md_table_init(pgd_t *pgd) 
		pmd_table = (pmd_t *)alloc_low_page();

		pmd_table = (pmd_t *)alloc_low_page();

		paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);

		paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);

		set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));

		set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));

		pud = pud_offset(pgd, 0);

		p4d = p4d_offset(pgd, 0);

		pud = pud_offset(p4d, 0);

		BUG_ON(pmd_table != pmd_offset(pud, 0));

		BUG_ON(pmd_table != pmd_offset(pud, 0));





		return pmd_table;

		return pmd_table;

	}

	}

#endif

#endif

	pud = pud_offset(pgd, 0);

	p4d = p4d_offset(pgd, 0);

	pud = pud_offset(p4d, 0);

	pmd_table = pmd_offset(pud, 0);

	pmd_table = pmd_offset(pud, 0);





	return pmd_table;

	return pmd_table;
@@ -390,8 +393,11 @@ pte_t *kmap_pte; 




static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr)

static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr)

{

{

	return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),

	pgd_t *pgd = pgd_offset_k(vaddr);

			vaddr), vaddr), vaddr);

	p4d_t *p4d = p4d_offset(pgd, vaddr);

	pud_t *pud = pud_offset(p4d, vaddr);

	pmd_t *pmd = pmd_offset(pud, vaddr);

	return pte_offset_kernel(pmd, vaddr);

}

}





static void __init kmap_init(void)

static void __init kmap_init(void)
@@ -410,6 +416,7 @@ static void __init permanent_kmaps_init(pgd_t *pgd_base) 
{

{

	unsigned long vaddr;

	unsigned long vaddr;

	pgd_t *pgd;

	pgd_t *pgd;

	p4d_t *p4d;

	pud_t *pud;

	pud_t *pud;

	pmd_t *pmd;

	pmd_t *pmd;

	pte_t *pte;

	pte_t *pte;
@@ -418,7 +425,8 @@ static void __init permanent_kmaps_init(pgd_t *pgd_base) 
	page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);

	page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);





	pgd = swapper_pg_dir + pgd_index(vaddr);

	pgd = swapper_pg_dir + pgd_index(vaddr);

	pud = pud_offset(pgd, vaddr);

	p4d = p4d_offset(pgd, vaddr);

	pud = pud_offset(p4d, vaddr);

	pmd = pmd_offset(pud, vaddr);

	pmd = pmd_offset(pud, vaddr);

	pte = pte_offset_kernel(pmd, vaddr);

	pte = pte_offset_kernel(pmd, vaddr);

	pkmap_page_table = pte;

	pkmap_page_table = pte;
@@ -450,6 +458,7 @@ void __init native_pagetable_init(void) 
{

{

	unsigned long pfn, va;

	unsigned long pfn, va;

	pgd_t *pgd, *base = swapper_pg_dir;

	pgd_t *pgd, *base = swapper_pg_dir;

	p4d_t *p4d;

	pud_t *pud;

	pud_t *pud;

	pmd_t *pmd;

	pmd_t *pmd;

	pte_t *pte;

	pte_t *pte;
@@ -469,7 +478,8 @@ void __init native_pagetable_init(void) 
		if (!pgd_present(*pgd))

		if (!pgd_present(*pgd))

			break;

			break;





		pud = pud_offset(pgd, va);

		p4d = p4d_offset(pgd, va);

		pud = pud_offset(p4d, va);

		pmd = pmd_offset(pud, va);

		pmd = pmd_offset(pud, va);

		if (!pmd_present(*pmd))

		if (!pmd_present(*pmd))

			break;

			break;

arch/x86/mm/ioremap.c

+2 −1
Original line number Original line Diff line number Diff line
@@ -425,7 +425,8 @@ static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) 
	/* Don't assume we're using swapper_pg_dir at this point */

	/* Don't assume we're using swapper_pg_dir at this point */

	pgd_t *base = __va(read_cr3());

	pgd_t *base = __va(read_cr3());

	pgd_t *pgd = &base[pgd_index(addr)];

	pgd_t *pgd = &base[pgd_index(addr)];

	pud_t *pud = pud_offset(pgd, addr);

	p4d_t *p4d = p4d_offset(pgd, addr);

	pud_t *pud = pud_offset(p4d, addr);

	pmd_t *pmd = pmd_offset(pud, addr);

	pmd_t *pmd = pmd_offset(pud, addr);





	return pmd;

	return pmd;