Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ac55c768 authored by David S. Miller's avatar David S. Miller
Browse files

sparc64: Switch to 4-level page tables. 



This has become necessary with chips that support more than 43-bits
of physical addressing.

Based almost entirely upon a patch by Bob Picco.

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
Acked-by: default avatarBob Picco <bob.picco@oracle.com>
parent 473ad7f4

arch/sparc/include/asm/page_64.h

+6 −0
Original line number Diff line number Diff line
@@ -57,18 +57,21 @@ void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topa 
typedef struct { unsigned long pte; } pte_t;

typedef struct { unsigned long iopte; } iopte_t;

typedef struct { unsigned long pmd; } pmd_t;

typedef struct { unsigned long pud; } pud_t;

typedef struct { unsigned long pgd; } pgd_t;

typedef struct { unsigned long pgprot; } pgprot_t;



#define pte_val(x)	((x).pte)

#define iopte_val(x)	((x).iopte)

#define pmd_val(x)      ((x).pmd)

#define pud_val(x)      ((x).pud)

#define pgd_val(x)	((x).pgd)

#define pgprot_val(x)	((x).pgprot)



#define __pte(x)	((pte_t) { (x) } )

#define __iopte(x)	((iopte_t) { (x) } )

#define __pmd(x)        ((pmd_t) { (x) } )

#define __pud(x)        ((pud_t) { (x) } )

#define __pgd(x)	((pgd_t) { (x) } )

#define __pgprot(x)	((pgprot_t) { (x) } )
@@ -77,18 +80,21 @@ typedef struct { unsigned long pgprot; } pgprot_t; 
typedef unsigned long pte_t;

typedef unsigned long iopte_t;

typedef unsigned long pmd_t;

typedef unsigned long pud_t;

typedef unsigned long pgd_t;

typedef unsigned long pgprot_t;



#define pte_val(x)	(x)

#define iopte_val(x)	(x)

#define pmd_val(x)      (x)

#define pud_val(x)      (x)

#define pgd_val(x)	(x)

#define pgprot_val(x)	(x)



#define __pte(x)	(x)

#define __iopte(x)	(x)

#define __pmd(x)        (x)

#define __pud(x)        (x)

#define __pgd(x)	(x)

#define __pgprot(x)	(x)

arch/sparc/include/asm/pgalloc_64.h

+27 −1
Original line number Diff line number Diff line
@@ -15,6 +15,13 @@ 


extern struct kmem_cache *pgtable_cache;



static inline void __pgd_populate(pgd_t *pgd, pud_t *pud)

{

	pgd_set(pgd, pud);

}



#define pgd_populate(MM, PGD, PUD)	__pgd_populate(PGD, PUD)



static inline pgd_t *pgd_alloc(struct mm_struct *mm)

{

	return kmem_cache_alloc(pgtable_cache, GFP_KERNEL);
@@ -25,7 +32,23 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) 
	kmem_cache_free(pgtable_cache, pgd);

}



#define pud_populate(MM, PUD, PMD)	pud_set(PUD, PMD)

static inline void __pud_populate(pud_t *pud, pmd_t *pmd)

{

	pud_set(pud, pmd);

}



#define pud_populate(MM, PUD, PMD)	__pud_populate(PUD, PMD)



static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)

{

	return kmem_cache_alloc(pgtable_cache,

				GFP_KERNEL|__GFP_REPEAT);

}



static inline void pud_free(struct mm_struct *mm, pud_t *pud)

{

	kmem_cache_free(pgtable_cache, pud);

}



static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)

{
@@ -91,4 +114,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte, 
#define __pmd_free_tlb(tlb, pmd, addr)		      \

	pgtable_free_tlb(tlb, pmd, false)



#define __pud_free_tlb(tlb, pud, addr)		      \

	pgtable_free_tlb(tlb, pud, false)



#endif /* _SPARC64_PGALLOC_H */

arch/sparc/include/asm/pgtable_64.h

+32 −5
Original line number Diff line number Diff line
@@ -20,8 +20,6 @@ 
#include <asm/page.h>

#include <asm/processor.h>



#include <asm-generic/pgtable-nopud.h>



/* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB).

 * The page copy blockops can use 0x6000000 to 0x8000000.

 * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range.
@@ -55,13 +53,21 @@ 
#define PMD_MASK	(~(PMD_SIZE-1))

#define PMD_BITS	(PAGE_SHIFT - 3)



/* PGDIR_SHIFT determines what a third-level page table entry can map */

#define PGDIR_SHIFT	(PAGE_SHIFT + (PAGE_SHIFT-3) + PMD_BITS)

/* PUD_SHIFT determines the size of the area a third-level page

 * table can map

 */

#define PUD_SHIFT	(PMD_SHIFT + PMD_BITS)

#define PUD_SIZE	(_AC(1,UL) << PUD_SHIFT)

#define PUD_MASK	(~(PUD_SIZE-1))

#define PUD_BITS	(PAGE_SHIFT - 3)



/* PGDIR_SHIFT determines what a fourth-level page table entry can map */

#define PGDIR_SHIFT	(PUD_SHIFT + PUD_BITS)

#define PGDIR_SIZE	(_AC(1,UL) << PGDIR_SHIFT)

#define PGDIR_MASK	(~(PGDIR_SIZE-1))

#define PGDIR_BITS	(PAGE_SHIFT - 3)



#if (PGDIR_SHIFT + PGDIR_BITS) != 43

#if (PGDIR_SHIFT + PGDIR_BITS) != 53

#error Page table parameters do not cover virtual address space properly.

#endif
@@ -93,6 +99,7 @@ static inline bool kern_addr_valid(unsigned long addr) 
/* Entries per page directory level. */

#define PTRS_PER_PTE	(1UL << (PAGE_SHIFT-3))

#define PTRS_PER_PMD	(1UL << PMD_BITS)

#define PTRS_PER_PUD	(1UL << PUD_BITS)

#define PTRS_PER_PGD	(1UL << PGDIR_BITS)



/* Kernel has a separate 44bit address space. */
@@ -101,6 +108,9 @@ static inline bool kern_addr_valid(unsigned long addr) 
#define pmd_ERROR(e)							\

	pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n",		\

	       __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0))

#define pud_ERROR(e)							\

	pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n",		\

	       __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0))

#define pgd_ERROR(e)							\

	pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n",		\

	       __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0))
@@ -779,6 +789,11 @@ static inline int pmd_present(pmd_t pmd) 
#define pud_bad(pud)			((pud_val(pud) & ~PAGE_MASK) || \

					 !__kern_addr_valid(pud_val(pud)))



#define pgd_none(pgd)			(!pgd_val(pgd))



#define pgd_bad(pgd)			((pgd_val(pgd) & ~PAGE_MASK) || \

					 !__kern_addr_valid(pgd_val(pgd)))



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

void set_pmd_at(struct mm_struct *mm, unsigned long addr,

		pmd_t *pmdp, pmd_t pmd);
@@ -815,10 +830,17 @@ static inline unsigned long __pmd_page(pmd_t pmd) 
#define pmd_clear(pmdp)			(pmd_val(*(pmdp)) = 0UL)

#define pud_present(pud)		(pud_val(pud) != 0U)

#define pud_clear(pudp)			(pud_val(*(pudp)) = 0UL)

#define pgd_page_vaddr(pgd)		\

	((unsigned long) __va(pgd_val(pgd)))

#define pgd_present(pgd)		(pgd_val(pgd) != 0U)

#define pgd_clear(pgdp)			(pgd_val(*(pgd)) = 0UL)



/* Same in both SUN4V and SUN4U.  */

#define pte_none(pte) 			(!pte_val(pte))



#define pgd_set(pgdp, pudp)	\

	(pgd_val(*(pgdp)) = (__pa((unsigned long) (pudp))))



/* to find an entry in a page-table-directory. */

#define pgd_index(address)	(((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))

#define pgd_offset(mm, address)	((mm)->pgd + pgd_index(address))
@@ -826,6 +848,11 @@ static inline unsigned long __pmd_page(pmd_t pmd) 
/* to find an entry in a kernel page-table-directory */

#define pgd_offset_k(address) pgd_offset(&init_mm, address)



/* Find an entry in the third-level page table.. */

#define pud_index(address)	(((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))

#define pud_offset(pgdp, address)	\

	((pud_t *) pgd_page_vaddr(*(pgdp)) + pud_index(address))



/* Find an entry in the second-level page table.. */

#define pmd_offset(pudp, address)	\

	((pmd_t *) pud_page_vaddr(*(pudp)) + \

arch/sparc/include/asm/tsb.h

+10 −0
Original line number Diff line number Diff line
@@ -144,6 +144,11 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; 
	srlx		REG2, 64 - PAGE_SHIFT, REG2; \

	andn		REG2, 0x7, REG2; \

	ldx		[REG1 + REG2], REG1; \

	brz,pn		REG1, FAIL_LABEL; \

	 sllx		VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \

	srlx		REG2, 64 - PAGE_SHIFT, REG2; \

	andn		REG2, 0x7, REG2; \

	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \

	brz,pn		REG1, FAIL_LABEL; \

	 sllx		VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \

	srlx		REG2, 64 - PAGE_SHIFT, REG2; \
@@ -197,6 +202,11 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; 
	srlx		REG2, 64 - PAGE_SHIFT, REG2; \

	andn		REG2, 0x7, REG2; \

	ldxa		[PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \

	brz,pn		REG1, FAIL_LABEL; \

	 sllx		VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \

	srlx		REG2, 64 - PAGE_SHIFT, REG2; \

	andn		REG2, 0x7, REG2; \

	ldxa		[REG1 + REG2] ASI_PHYS_USE_EC, REG1; \

	brz,pn		REG1, FAIL_LABEL; \

	 sllx		VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \

	srlx		REG2, 64 - PAGE_SHIFT, REG2; \

arch/sparc/kernel/smp_64.c

+7 −0
Original line number Diff line number Diff line
@@ -1467,6 +1467,13 @@ static void __init pcpu_populate_pte(unsigned long addr) 
	pud_t *pud;

	pmd_t *pmd;



	if (pgd_none(*pgd)) {

		pud_t *new;



		new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

		pgd_populate(&init_mm, pgd, new);

	}



	pud = pud_offset(pgd, addr);

	if (pud_none(*pud)) {

		pmd_t *new;