Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c2febafc authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds
Browse files

mm: convert generic code to 5-level paging 



Convert all non-architecture-specific code to 5-level paging.

It's mostly mechanical adding handling one more page table level in
places where we deal with pud_t.

Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 048456dc

drivers/misc/sgi-gru/grufault.c

+7 −2
Original line number Diff line number Diff line
@@ -219,15 +219,20 @@ static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, 
	int write, unsigned long *paddr, int *pageshift)

{

	pgd_t *pgdp;

	pmd_t *pmdp;

	p4d_t *p4dp;

	pud_t *pudp;

	pmd_t *pmdp;

	pte_t pte;



	pgdp = pgd_offset(vma->vm_mm, vaddr);

	if (unlikely(pgd_none(*pgdp)))

		goto err;



	pudp = pud_offset(pgdp, vaddr);

	p4dp = p4d_offset(pgdp, vaddr);

	if (unlikely(p4d_none(*p4dp)))

		goto err;



	pudp = pud_offset(p4dp, vaddr);

	if (unlikely(pud_none(*pudp)))

		goto err;

fs/userfaultfd.c

+5 −1
Original line number Diff line number Diff line
@@ -267,6 +267,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, 
{

	struct mm_struct *mm = ctx->mm;

	pgd_t *pgd;

	p4d_t *p4d;

	pud_t *pud;

	pmd_t *pmd, _pmd;

	pte_t *pte;
@@ -277,7 +278,10 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, 
	pgd = pgd_offset(mm, address);

	if (!pgd_present(*pgd))

		goto out;

	pud = pud_offset(pgd, address);

	p4d = p4d_offset(pgd, address);

	if (!p4d_present(*p4d))

		goto out;

	pud = pud_offset(p4d, address);

	if (!pud_present(*pud))

		goto out;

	pmd = pmd_offset(pud, address);

include/asm-generic/pgtable.h

+45 −3
Original line number Diff line number Diff line
@@ -10,9 +10,9 @@ 
#include <linux/bug.h>

#include <linux/errno.h>



#if 4 - defined(__PAGETABLE_PUD_FOLDED) - defined(__PAGETABLE_PMD_FOLDED) != \

	CONFIG_PGTABLE_LEVELS

#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{PUD,PMD}_FOLDED

#if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \

	defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS

#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED

#endif



/*
@@ -424,6 +424,13 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 
	(__boundary - 1 < (end) - 1)? __boundary: (end);		\

})



#ifndef p4d_addr_end

#define p4d_addr_end(addr, end)						\

({	unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK;	\

	(__boundary - 1 < (end) - 1)? __boundary: (end);		\

})

#endif



#ifndef pud_addr_end

#define pud_addr_end(addr, end)						\

({	unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK;	\
@@ -444,6 +451,7 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 
 * Do the tests inline, but report and clear the bad entry in mm/memory.c.

 */

void pgd_clear_bad(pgd_t *);

void p4d_clear_bad(p4d_t *);

void pud_clear_bad(pud_t *);

void pmd_clear_bad(pmd_t *);
@@ -458,6 +466,17 @@ static inline int pgd_none_or_clear_bad(pgd_t *pgd) 
	return 0;

}



static inline int p4d_none_or_clear_bad(p4d_t *p4d)

{

	if (p4d_none(*p4d))

		return 1;

	if (unlikely(p4d_bad(*p4d))) {

		p4d_clear_bad(p4d);

		return 1;

	}

	return 0;

}



static inline int pud_none_or_clear_bad(pud_t *pud)

{

	if (pud_none(*pud))
@@ -844,11 +863,30 @@ static inline int pmd_protnone(pmd_t pmd) 
#endif /* CONFIG_MMU */



#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP



#ifndef __PAGETABLE_P4D_FOLDED

int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot);

int p4d_clear_huge(p4d_t *p4d);

#else

static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)

{

	return 0;

}

static inline int p4d_clear_huge(p4d_t *p4d)

{

	return 0;

}

#endif /* !__PAGETABLE_P4D_FOLDED */



int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);

int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);

int pud_clear_huge(pud_t *pud);

int pmd_clear_huge(pmd_t *pmd);

#else	/* !CONFIG_HAVE_ARCH_HUGE_VMAP */

static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)

{

	return 0;

}

static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)

{

	return 0;
@@ -857,6 +895,10 @@ static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) 
{

	return 0;

}

static inline int p4d_clear_huge(p4d_t *p4d)

{

	return 0;

}

static inline int pud_clear_huge(pud_t *pud)

{

	return 0;

include/linux/hugetlb.h

+4 −1
Original line number Diff line number Diff line
@@ -122,7 +122,7 @@ struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, 
struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,

				pud_t *pud, int flags);

int pmd_huge(pmd_t pmd);

int pud_huge(pud_t pmd);

int pud_huge(pud_t pud);

unsigned long hugetlb_change_protection(struct vm_area_struct *vma,

		unsigned long address, unsigned long end, pgprot_t newprot);
@@ -197,6 +197,9 @@ static inline void __unmap_hugepage_range(struct mmu_gather *tlb, 
#ifndef pgd_huge

#define pgd_huge(x)	0

#endif

#ifndef p4d_huge

#define p4d_huge(x)	0

#endif



#ifndef pgd_write

static inline int pgd_write(pgd_t pgd)

include/linux/kasan.h

+1 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ extern unsigned char kasan_zero_page[PAGE_SIZE]; 
extern pte_t kasan_zero_pte[PTRS_PER_PTE];

extern pmd_t kasan_zero_pmd[PTRS_PER_PMD];

extern pud_t kasan_zero_pud[PTRS_PER_PUD];

extern p4d_t kasan_zero_p4d[PTRS_PER_P4D];



void kasan_populate_zero_shadow(const void *shadow_start,

				const void *shadow_end);