Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d08de8e2 authored by Gerald Schaefer's avatar Gerald Schaefer Committed by Martin Schwidefsky
Browse files

s390/mm: add support for 2GB hugepages 



This adds support for 2GB hugetlbfs pages on s390.

Reviewed-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: default avatarGerald Schaefer <gerald.schaefer@de.ibm.com>
Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent 46210c44

arch/s390/include/asm/hugetlb.h

+4 −1
Original line number Diff line number Diff line
@@ -41,6 +41,9 @@ static inline int prepare_hugepage_range(struct file *file, 
static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,

				  pte_t *ptep)

{

	if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)

		pte_val(*ptep) = _REGION3_ENTRY_EMPTY;

	else

		pte_val(*ptep) = _SEGMENT_ENTRY_EMPTY;

}

arch/s390/include/asm/page.h

+1 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 
#define HPAGE_SIZE	(1UL << HPAGE_SHIFT)

#define HPAGE_MASK	(~(HPAGE_SIZE - 1))

#define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)

#define HUGE_MAX_HSTATE		2



#define ARCH_HAS_SETCLEAR_HUGE_PTE

#define ARCH_HAS_HUGE_PTE_TYPE

arch/s390/include/asm/pgtable.h

+49 −22
Original line number Diff line number Diff line
@@ -306,6 +306,9 @@ static inline int is_module_addr(void *addr) 
#define _REGION3_ENTRY_SOFT_DIRTY 0x0000 /* SW region soft dirty bit */

#endif



#define _REGION_ENTRY_BITS	 0xfffffffffffff227UL

#define _REGION_ENTRY_BITS_LARGE 0xffffffff8000fe27UL



/* Bits in the segment table entry */

#define _SEGMENT_ENTRY_BITS	0xfffffffffffffe33UL

#define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff0ff33UL
@@ -573,7 +576,7 @@ static inline int pud_none(pud_t pud) 
{

	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)

		return 0;

	return (pud_val(pud) & _REGION_ENTRY_INVALID) != 0UL;

	return pud_val(pud) == _REGION3_ENTRY_EMPTY;

}



static inline int pud_large(pud_t pud)
@@ -593,17 +596,25 @@ static inline unsigned long pud_pfn(pud_t pud) 
	return (pud_val(pud) & origin_mask) >> PAGE_SHIFT;

}



static inline int pmd_large(pmd_t pmd)

{

	return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0;

}



static inline int pmd_bad(pmd_t pmd)

{

	if (pmd_large(pmd))

		return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS_LARGE) != 0;

	return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0;

}



static inline int pud_bad(pud_t pud)

{

	/*

	 * With dynamic page table levels the pud can be a region table

	 * entry or a segment table entry. Check for the bit that are

	 * invalid for either table entry.

	 */

	unsigned long mask =

		~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID &

		~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;

	return (pud_val(pud) & mask) != 0;

	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)

		return pmd_bad(__pmd(pud_val(pud)));

	if (pud_large(pud))

		return (pud_val(pud) & ~_REGION_ENTRY_BITS_LARGE) != 0;

	return (pud_val(pud) & ~_REGION_ENTRY_BITS) != 0;

}



static inline int pmd_present(pmd_t pmd)
@@ -616,11 +627,6 @@ static inline int pmd_none(pmd_t pmd) 
	return pmd_val(pmd) == _SEGMENT_ENTRY_INVALID;

}



static inline int pmd_large(pmd_t pmd)

{

	return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0;

}



static inline unsigned long pmd_pfn(pmd_t pmd)

{

	unsigned long origin_mask;
@@ -631,13 +637,6 @@ static inline unsigned long pmd_pfn(pmd_t pmd) 
	return (pmd_val(pmd) & origin_mask) >> PAGE_SHIFT;

}



static inline int pmd_bad(pmd_t pmd)

{

	if (pmd_large(pmd))

		return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS_LARGE) != 0;

	return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0;

}



#define __HAVE_ARCH_PMD_WRITE

static inline int pmd_write(pmd_t pmd)

{
@@ -1081,6 +1080,7 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) 
#define pte_page(x) pfn_to_page(pte_pfn(x))



#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))

#define pud_page(pud) pfn_to_page(pud_pfn(pud))



/* Find an entry in the lowest level page table.. */

#define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr))
@@ -1238,6 +1238,19 @@ static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp) 
		: "cc" );

}



static inline void __pudp_idte(unsigned long address, pud_t *pudp)

{

	unsigned long r3o;



	r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t);

	r3o |= _ASCE_TYPE_REGION3;

	asm volatile(

		"	.insn	rrf,0xb98e0000,%2,%3,0,0"

		: "=m" (*pudp)

		: "m" (*pudp), "a" (r3o), "a" ((address & PUD_MASK))

		: "cc");

}



static inline void __pmdp_idte_local(unsigned long address, pmd_t *pmdp)

{

	unsigned long sto;
@@ -1250,8 +1263,22 @@ static inline void __pmdp_idte_local(unsigned long address, pmd_t *pmdp) 
		: "cc" );

}



static inline void __pudp_idte_local(unsigned long address, pud_t *pudp)

{

	unsigned long r3o;



	r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t);

	r3o |= _ASCE_TYPE_REGION3;

	asm volatile(

		"	.insn	rrf,0xb98e0000,%2,%3,0,1"

		: "=m" (*pudp)

		: "m" (*pudp), "a" (r3o), "a" ((address & PUD_MASK))

		: "cc");

}



pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);

pmd_t pmdp_xchg_lazy(struct mm_struct *, unsigned long, pmd_t *, pmd_t);

pud_t pudp_xchg_direct(struct mm_struct *, unsigned long, pud_t *, pud_t);



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

arch/s390/mm/gmap.c

+3 −0
Original line number Diff line number Diff line
@@ -430,6 +430,9 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) 
	VM_BUG_ON(pgd_none(*pgd));

	pud = pud_offset(pgd, vmaddr);

	VM_BUG_ON(pud_none(*pud));

	/* large puds cannot yet be handled */

	if (pud_large(*pud))

		return -EFAULT;

	pmd = pmd_offset(pud, vmaddr);

	VM_BUG_ON(pmd_none(*pmd));

	/* large pmds cannot yet be handled */

arch/s390/mm/gup.c

+44 −1
Original line number Diff line number Diff line
@@ -128,6 +128,44 @@ static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, 
	return 1;

}



static int gup_huge_pud(pud_t *pudp, pud_t pud, unsigned long addr,

		unsigned long end, int write, struct page **pages, int *nr)

{

	struct page *head, *page;

	unsigned long mask;

	int refs;



	mask = (write ? _REGION_ENTRY_PROTECT : 0) | _REGION_ENTRY_INVALID;

	if ((pud_val(pud) & mask) != 0)

		return 0;

	VM_BUG_ON(!pfn_valid(pud_pfn(pud)));



	refs = 0;

	head = pud_page(pud);

	page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);

	do {

		VM_BUG_ON_PAGE(compound_head(page) != head, page);

		pages[*nr] = page;

		(*nr)++;

		page++;

		refs++;

	} while (addr += PAGE_SIZE, addr != end);



	if (!page_cache_add_speculative(head, refs)) {

		*nr -= refs;

		return 0;

	}



	if (unlikely(pud_val(pud) != pud_val(*pudp))) {

		*nr -= refs;

		while (refs--)

			put_page(head);

		return 0;

	}



	return 1;

}



static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr,

		unsigned long end, int write, struct page **pages, int *nr)

{
@@ -144,7 +182,12 @@ static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, 
		next = pud_addr_end(addr, end);

		if (pud_none(pud))

			return 0;

		if (!gup_pmd_range(pudp, pud, addr, next, write, pages, nr))

		if (unlikely(pud_large(pud))) {

			if (!gup_huge_pud(pudp, pud, addr, next, write, pages,

					  nr))

				return 0;

		} else if (!gup_pmd_range(pudp, pud, addr, next, write, pages,

					  nr))

			return 0;

	} while (pudp++, addr = next, addr != end);