Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f2d6bfe9 authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds
Browse files

thp: add x86 32bit support 



Add support for transparent hugepages to x86 32bit.

Share the same VM_ bitflag for VM_MAPPED_COPY.  mm/nommu.c will never
support transparent hugepages.

Signed-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Signed-off-by: default avatarAndrea Arcangeli <aarcange@redhat.com>
Reviewed-by: default avatarRik van Riel <riel@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 5f24ce5f

arch/x86/include/asm/pgtable-2level.h

+9 −0
Original line number Diff line number Diff line
@@ -46,6 +46,15 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp) 
#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)

#endif



#ifdef CONFIG_SMP

static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)

{

	return __pmd(xchg((pmdval_t *)xp, 0));

}

#else

#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)

#endif



/*

 * Bits _PAGE_BIT_PRESENT, _PAGE_BIT_FILE and _PAGE_BIT_PROTNONE are taken,

 * split up the 29 bits of offset into this range:

arch/x86/include/asm/pgtable-3level.h

+23 −0
Original line number Diff line number Diff line
@@ -104,6 +104,29 @@ static inline pte_t native_ptep_get_and_clear(pte_t *ptep) 
#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)

#endif



#ifdef CONFIG_SMP

union split_pmd {

	struct {

		u32 pmd_low;

		u32 pmd_high;

	};

	pmd_t pmd;

};

static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)

{

	union split_pmd res, *orig = (union split_pmd *)pmdp;



	/* xchg acts as a barrier before setting of the high bits */

	res.pmd_low = xchg(&orig->pmd_low, 0);

	res.pmd_high = orig->pmd_high;

	orig->pmd_high = 0;



	return res.pmd;

}

#else

#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)

#endif



/*

 * Bits 0, 6 and 7 are taken in the low part of the pte,

 * put the 32 bits of offset into the high part.

arch/x86/include/asm/pgtable.h

+117 −0
Original line number Diff line number Diff line
@@ -97,6 +97,11 @@ static inline int pte_young(pte_t pte) 
	return pte_flags(pte) & _PAGE_ACCESSED;

}



static inline int pmd_young(pmd_t pmd)

{

	return pmd_flags(pmd) & _PAGE_ACCESSED;

}



static inline int pte_write(pte_t pte)

{

	return pte_flags(pte) & _PAGE_RW;
@@ -145,6 +150,18 @@ static inline int pmd_large(pmd_t pte) 
		(_PAGE_PSE | _PAGE_PRESENT);

}



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

static inline int pmd_trans_splitting(pmd_t pmd)

{

	return pmd_val(pmd) & _PAGE_SPLITTING;

}



static inline int pmd_trans_huge(pmd_t pmd)

{

	return pmd_val(pmd) & _PAGE_PSE;

}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */



static inline pte_t pte_set_flags(pte_t pte, pteval_t set)

{

	pteval_t v = native_pte_val(pte);
@@ -219,6 +236,55 @@ static inline pte_t pte_mkspecial(pte_t pte) 
	return pte_set_flags(pte, _PAGE_SPECIAL);

}



static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)

{

	pmdval_t v = native_pmd_val(pmd);



	return __pmd(v | set);

}



static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)

{

	pmdval_t v = native_pmd_val(pmd);



	return __pmd(v & ~clear);

}



static inline pmd_t pmd_mkold(pmd_t pmd)

{

	return pmd_clear_flags(pmd, _PAGE_ACCESSED);

}



static inline pmd_t pmd_wrprotect(pmd_t pmd)

{

	return pmd_clear_flags(pmd, _PAGE_RW);

}



static inline pmd_t pmd_mkdirty(pmd_t pmd)

{

	return pmd_set_flags(pmd, _PAGE_DIRTY);

}



static inline pmd_t pmd_mkhuge(pmd_t pmd)

{

	return pmd_set_flags(pmd, _PAGE_PSE);

}



static inline pmd_t pmd_mkyoung(pmd_t pmd)

{

	return pmd_set_flags(pmd, _PAGE_ACCESSED);

}



static inline pmd_t pmd_mkwrite(pmd_t pmd)

{

	return pmd_set_flags(pmd, _PAGE_RW);

}



static inline pmd_t pmd_mknotpresent(pmd_t pmd)

{

	return pmd_clear_flags(pmd, _PAGE_PRESENT);

}



/*

 * Mask out unsupported bits in a present pgprot.  Non-present pgprots

 * can use those bits for other purposes, so leave them be.
@@ -527,6 +593,14 @@ static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep) 
	return res;

}



static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)

{

	pmd_t res = *pmdp;



	native_pmd_clear(pmdp);

	return res;

}



static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,

				     pte_t *ptep , pte_t pte)

{
@@ -616,6 +690,49 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, 


#define flush_tlb_fix_spurious_fault(vma, address)



#define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))



#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS

extern int pmdp_set_access_flags(struct vm_area_struct *vma,

				 unsigned long address, pmd_t *pmdp,

				 pmd_t entry, int dirty);



#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG

extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,

				     unsigned long addr, pmd_t *pmdp);



#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH

extern int pmdp_clear_flush_young(struct vm_area_struct *vma,

				  unsigned long address, pmd_t *pmdp);





#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH

extern void pmdp_splitting_flush(struct vm_area_struct *vma,

				 unsigned long addr, pmd_t *pmdp);



#define __HAVE_ARCH_PMD_WRITE

static inline int pmd_write(pmd_t pmd)

{

	return pmd_flags(pmd) & _PAGE_RW;

}



#define __HAVE_ARCH_PMDP_GET_AND_CLEAR

static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, unsigned long addr,

				       pmd_t *pmdp)

{

	pmd_t pmd = native_pmdp_get_and_clear(pmdp);

	pmd_update(mm, addr, pmdp);

	return pmd;

}



#define __HAVE_ARCH_PMDP_SET_WRPROTECT

static inline void pmdp_set_wrprotect(struct mm_struct *mm,

				      unsigned long addr, pmd_t *pmdp)

{

	clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);

	pmd_update(mm, addr, pmdp);

}



/*

 * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);

 *

arch/x86/include/asm/pgtable_64.h

+0 −109
Original line number Diff line number Diff line
@@ -182,115 +182,6 @@ extern void cleanup_highmap(void); 


#define __HAVE_ARCH_PTE_SAME



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

static inline int pmd_trans_splitting(pmd_t pmd)

{

	return pmd_val(pmd) & _PAGE_SPLITTING;

}



static inline int pmd_trans_huge(pmd_t pmd)

{

	return pmd_val(pmd) & _PAGE_PSE;

}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */



#define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))



#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS

extern int pmdp_set_access_flags(struct vm_area_struct *vma,

				 unsigned long address, pmd_t *pmdp,

				 pmd_t entry, int dirty);



#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG

extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,

				     unsigned long addr, pmd_t *pmdp);



#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH

extern int pmdp_clear_flush_young(struct vm_area_struct *vma,

				  unsigned long address, pmd_t *pmdp);





#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH

extern void pmdp_splitting_flush(struct vm_area_struct *vma,

				 unsigned long addr, pmd_t *pmdp);



#define __HAVE_ARCH_PMD_WRITE

static inline int pmd_write(pmd_t pmd)

{

	return pmd_flags(pmd) & _PAGE_RW;

}



#define __HAVE_ARCH_PMDP_GET_AND_CLEAR

static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, unsigned long addr,

				       pmd_t *pmdp)

{

	pmd_t pmd = native_pmdp_get_and_clear(pmdp);

	pmd_update(mm, addr, pmdp);

	return pmd;

}



#define __HAVE_ARCH_PMDP_SET_WRPROTECT

static inline void pmdp_set_wrprotect(struct mm_struct *mm,

				      unsigned long addr, pmd_t *pmdp)

{

	clear_bit(_PAGE_BIT_RW, (unsigned long *)&pmdp->pmd);

	pmd_update(mm, addr, pmdp);

}



static inline int pmd_young(pmd_t pmd)

{

	return pmd_flags(pmd) & _PAGE_ACCESSED;

}



static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)

{

	pmdval_t v = native_pmd_val(pmd);



	return native_make_pmd(v | set);

}



static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)

{

	pmdval_t v = native_pmd_val(pmd);



	return native_make_pmd(v & ~clear);

}



static inline pmd_t pmd_mkold(pmd_t pmd)

{

	return pmd_clear_flags(pmd, _PAGE_ACCESSED);

}



static inline pmd_t pmd_wrprotect(pmd_t pmd)

{

	return pmd_clear_flags(pmd, _PAGE_RW);

}



static inline pmd_t pmd_mkdirty(pmd_t pmd)

{

	return pmd_set_flags(pmd, _PAGE_DIRTY);

}



static inline pmd_t pmd_mkhuge(pmd_t pmd)

{

	return pmd_set_flags(pmd, _PAGE_PSE);

}



static inline pmd_t pmd_mkyoung(pmd_t pmd)

{

	return pmd_set_flags(pmd, _PAGE_ACCESSED);

}



static inline pmd_t pmd_mkwrite(pmd_t pmd)

{

	return pmd_set_flags(pmd, _PAGE_RW);

}



static inline pmd_t pmd_mknotpresent(pmd_t pmd)

{

	return pmd_clear_flags(pmd, _PAGE_PRESENT);

}



#endif /* !__ASSEMBLY__ */



#endif /* _ASM_X86_PGTABLE_64_H */

arch/x86/mm/pgtable.c

+2 −2
Original line number Diff line number Diff line
@@ -362,7 +362,7 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, 


	if (pmd_young(*pmdp))

		ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,

					 (unsigned long *) &pmdp->pmd);

					 (unsigned long *)pmdp);



	if (ret)

		pmd_update(vma->vm_mm, addr, pmdp);
@@ -404,7 +404,7 @@ void pmdp_splitting_flush(struct vm_area_struct *vma, 
	int set;

	VM_BUG_ON(address & ~HPAGE_PMD_MASK);

	set = !test_and_set_bit(_PAGE_BIT_SPLITTING,

				(unsigned long *)&pmdp->pmd);

				(unsigned long *)pmdp);

	if (set) {

		pmd_update(vma->vm_mm, address, pmdp);

		/* need tlb flush only to serialize against gup-fast */