powerpc/mm: Move hash64 PTE bits from book3s/64/pgtable.h to hash.h (371352ca) · Commits · e / devices / android_kernel_fairphone_FP3

arch/powerpc/include/asm/book3s/64/hash.h

+445 −3

Original line number	Diff line number	Diff line
		@@ -2,6 +2,61 @@
		#define _ASM_POWERPC_BOOK3S_64_HASH_H
		#ifdef __KERNEL__

		#ifdef CONFIG_PPC_64K_PAGES
		#include <asm/book3s/64/hash-64k.h>
		#else
		#include <asm/book3s/64/hash-4k.h>
		#endif

		/*
		* Size of EA range mapped by our pagetables.
		*/
		#define PGTABLE_EADDR_SIZE (PTE_INDEX_SIZE + PMD_INDEX_SIZE + \
		PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT)
		#define PGTABLE_RANGE (ASM_CONST(1) << PGTABLE_EADDR_SIZE)

		#ifdef CONFIG_TRANSPARENT_HUGEPAGE
		#define PMD_CACHE_INDEX (PMD_INDEX_SIZE + 1)
		#else
		#define PMD_CACHE_INDEX PMD_INDEX_SIZE
		#endif
		/*
		* Define the address range of the kernel non-linear virtual area
		*/
		#define KERN_VIRT_START ASM_CONST(0xD000000000000000)
		#define KERN_VIRT_SIZE ASM_CONST(0x0000100000000000)

		/*
		* The vmalloc space starts at the beginning of that region, and
		* occupies half of it on hash CPUs and a quarter of it on Book3E
		* (we keep a quarter for the virtual memmap)
		*/
		#define VMALLOC_START KERN_VIRT_START
		#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
		#define VMALLOC_END (VMALLOC_START + VMALLOC_SIZE)

		/*
		* Region IDs
		*/
		#define REGION_SHIFT 60UL
		#define REGION_MASK (0xfUL << REGION_SHIFT)
		#define REGION_ID(ea) (((unsigned long)(ea)) >> REGION_SHIFT)

		#define VMALLOC_REGION_ID (REGION_ID(VMALLOC_START))
		#define KERNEL_REGION_ID (REGION_ID(PAGE_OFFSET))
		#define VMEMMAP_REGION_ID (0xfUL) /* Server only */
		#define USER_REGION_ID (0UL)

		/*
		* Defines the address of the vmemap area, in its own region on
		* hash table CPUs.
		*/
		#define VMEMMAP_BASE (VMEMMAP_REGION_ID << REGION_SHIFT)

		#ifdef CONFIG_PPC_MM_SLICES
		#define HAVE_ARCH_UNMAPPED_AREA
		#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
		#endif /* CONFIG_PPC_MM_SLICES */
		/*
		* Common bits between 4K and 64K pages in a linux-style PTE.
		* These match the bits in the (hardware-defined) PowerPC PTE as closely
		@@ -46,11 +101,398 @@
		/* Hash table based platforms need atomic updates of the linux PTE */
		#define PTE_ATOMIC_UPDATES 1

		#ifdef CONFIG_PPC_64K_PAGES
		#include <asm/book3s/64/hash-64k.h>
		/*
		* THP pages can't be special. So use the _PAGE_SPECIAL
		*/
		#define _PAGE_SPLITTING _PAGE_SPECIAL

		/*
		* We need to differentiate between explicit huge page and THP huge
		* page, since THP huge page also need to track real subpage details
		*/
		#define _PAGE_THP_HUGE _PAGE_4K_PFN

		/*
		* set of bits not changed in pmd_modify.
		*/
		#define _HPAGE_CHG_MASK (PTE_RPN_MASK \| _PAGE_HPTEFLAGS \| \
		_PAGE_DIRTY \| _PAGE_ACCESSED \| _PAGE_SPLITTING \| \
		_PAGE_THP_HUGE)
		#define _PTE_NONE_MASK _PAGE_HPTEFLAGS
		/*
		* The mask convered by the RPN must be a ULL on 32-bit platforms with
		* 64-bit PTEs
		*/
		#define PTE_RPN_MASK (~((1UL << PTE_RPN_SHIFT) - 1))
		/*
		* _PAGE_CHG_MASK masks of bits that are to be preserved across
		* pgprot changes
		*/
		#define _PAGE_CHG_MASK (PTE_RPN_MASK \| _PAGE_HPTEFLAGS \| _PAGE_DIRTY \| \
		_PAGE_ACCESSED \| _PAGE_SPECIAL)
		/*
		* Mask of bits returned by pte_pgprot()
		*/
		#define PAGE_PROT_BITS (_PAGE_GUARDED \| _PAGE_COHERENT \| _PAGE_NO_CACHE \| \
		_PAGE_WRITETHRU \| _PAGE_4K_PFN \| \
		_PAGE_USER \| _PAGE_ACCESSED \| \
		_PAGE_RW \| _PAGE_DIRTY \| _PAGE_EXEC)
		/*
		* We define 2 sets of base prot bits, one for basic pages (ie,
		* cacheable kernel and user pages) and one for non cacheable
		* pages. We always set _PAGE_COHERENT when SMP is enabled or
		* the processor might need it for DMA coherency.
		*/
		#define _PAGE_BASE_NC (_PAGE_PRESENT \| _PAGE_ACCESSED \| _PAGE_PSIZE)
		#define _PAGE_BASE (_PAGE_BASE_NC \| _PAGE_COHERENT)

		/* Permission masks used to generate the __P and __S table,
		*
		* Note:__pgprot is defined in arch/powerpc/include/asm/page.h
		*
		* Write permissions imply read permissions for now (we could make write-only
		* pages on BookE but we don't bother for now). Execute permission control is
		* possible on platforms that define _PAGE_EXEC
		*
		* Note due to the way vm flags are laid out, the bits are XWR
		*/
		#define PAGE_NONE __pgprot(_PAGE_BASE)
		#define PAGE_SHARED __pgprot(_PAGE_BASE \| _PAGE_USER \| _PAGE_RW)
		#define PAGE_SHARED_X __pgprot(_PAGE_BASE \| _PAGE_USER \| _PAGE_RW \| \
		_PAGE_EXEC)
		#define PAGE_COPY __pgprot(_PAGE_BASE \| _PAGE_USER )
		#define PAGE_COPY_X __pgprot(_PAGE_BASE \| _PAGE_USER \| _PAGE_EXEC)
		#define PAGE_READONLY __pgprot(_PAGE_BASE \| _PAGE_USER )
		#define PAGE_READONLY_X __pgprot(_PAGE_BASE \| _PAGE_USER \| _PAGE_EXEC)

		#define __P000 PAGE_NONE
		#define __P001 PAGE_READONLY
		#define __P010 PAGE_COPY
		#define __P011 PAGE_COPY
		#define __P100 PAGE_READONLY_X
		#define __P101 PAGE_READONLY_X
		#define __P110 PAGE_COPY_X
		#define __P111 PAGE_COPY_X

		#define __S000 PAGE_NONE
		#define __S001 PAGE_READONLY
		#define __S010 PAGE_SHARED
		#define __S011 PAGE_SHARED
		#define __S100 PAGE_READONLY_X
		#define __S101 PAGE_READONLY_X
		#define __S110 PAGE_SHARED_X
		#define __S111 PAGE_SHARED_X

		/* Permission masks used for kernel mappings */
		#define PAGE_KERNEL __pgprot(_PAGE_BASE \| _PAGE_KERNEL_RW)
		#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC \| _PAGE_KERNEL_RW \| \
		_PAGE_NO_CACHE)
		#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC \| _PAGE_KERNEL_RW \| \
		_PAGE_NO_CACHE \| _PAGE_GUARDED)
		#define PAGE_KERNEL_X __pgprot(_PAGE_BASE \| _PAGE_KERNEL_RWX)
		#define PAGE_KERNEL_RO __pgprot(_PAGE_BASE \| _PAGE_KERNEL_RO)
		#define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE \| _PAGE_KERNEL_ROX)

		/* Protection used for kernel text. We want the debuggers to be able to
		* set breakpoints anywhere, so don't write protect the kernel text
		* on platforms where such control is possible.
		*/
		#if defined(CONFIG_KGDB) \|\| defined(CONFIG_XMON) \|\| defined(CONFIG_BDI_SWITCH) \|\|\
		defined(CONFIG_KPROBES) \|\| defined(CONFIG_DYNAMIC_FTRACE)
		#define PAGE_KERNEL_TEXT PAGE_KERNEL_X
		#else
		#include <asm/book3s/64/hash-4k.h>
		#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX
		#endif

		/* Make modules code happy. We don't set RO yet */
		#define PAGE_KERNEL_EXEC PAGE_KERNEL_X
		#define PAGE_AGP (PAGE_KERNEL_NC)

		#define PMD_BAD_BITS (PTE_TABLE_SIZE-1)
		#define PUD_BAD_BITS (PMD_TABLE_SIZE-1)
		/*
		* We save the slot number & secondary bit in the second half of the
		* PTE page. We use the 8 bytes per each pte entry.
		*/
		#define PTE_PAGE_HIDX_OFFSET (PTRS_PER_PTE * 8)

		#ifndef __ASSEMBLY__
		#define pmd_bad(pmd) (!is_kernel_addr(pmd_val(pmd)) \
		\|\| (pmd_val(pmd) & PMD_BAD_BITS))
		#define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS)

		#define pud_bad(pud) (!is_kernel_addr(pud_val(pud)) \
		\|\| (pud_val(pud) & PUD_BAD_BITS))
		#define pud_page_vaddr(pud) (pud_val(pud) & ~PUD_MASKED_BITS)

		#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))
		#define pmd_index(address) (((address) >> (PMD_SHIFT)) & (PTRS_PER_PMD - 1))
		#define pte_index(address) (((address) >> (PAGE_SHIFT)) & (PTRS_PER_PTE - 1))

		extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
		pte_t *ptep, unsigned long pte, int huge);
		extern unsigned long pmd_hugepage_update(struct mm_struct *mm,
		unsigned long addr,
		pmd_t *pmdp,
		unsigned long clr,
		unsigned long set);
		/* Atomic PTE updates */
		static inline unsigned long pte_update(struct mm_struct *mm,
		unsigned long addr,
		pte_t *ptep, unsigned long clr,
		unsigned long set,
		int huge)
		{
		unsigned long old, tmp;

		__asm__ __volatile__(
		"1: ldarx %0,0,%3 # pte_update\n\
		andi. %1,%0,%6\n\
		bne- 1b \n\
		andc %1,%0,%4 \n\
		or %1,%1,%7\n\
		stdcx. %1,0,%3 \n\
		bne- 1b"
		: "=&r" (old), "=&r" (tmp), "=m" (*ptep)
		: "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY), "r" (set)
		: "cc" );
		/* huge pages use the old page table lock */
		if (!huge)
		assert_pte_locked(mm, addr);

		if (old & _PAGE_HASHPTE)
		hpte_need_flush(mm, addr, ptep, old, huge);

		return old;
		}

		static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
		unsigned long addr, pte_t *ptep)
		{
		unsigned long old;

		if ((pte_val(*ptep) & (_PAGE_ACCESSED \| _PAGE_HASHPTE)) == 0)
		return 0;
		old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
		return (old & _PAGE_ACCESSED) != 0;
		}
		#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
		#define ptep_test_and_clear_young(__vma, __addr, __ptep) \
		({ \
		int __r; \
		__r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \
		__r; \
		})

		#define __HAVE_ARCH_PTEP_SET_WRPROTECT
		static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
		pte_t *ptep)
		{

		if ((pte_val(*ptep) & _PAGE_RW) == 0)
		return;

		pte_update(mm, addr, ptep, _PAGE_RW, 0, 0);
		}

		static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
		unsigned long addr, pte_t *ptep)
		{
		if ((pte_val(*ptep) & _PAGE_RW) == 0)
		return;

		pte_update(mm, addr, ptep, _PAGE_RW, 0, 1);
		}

		/*
		* We currently remove entries from the hashtable regardless of whether
		* the entry was young or dirty. The generic routines only flush if the
		* entry was young or dirty which is not good enough.
		*
		* We should be more intelligent about this but for the moment we override
		* these functions and force a tlb flush unconditionally
		*/
		#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
		#define ptep_clear_flush_young(__vma, __address, __ptep) \
		({ \
		int __young = __ptep_test_and_clear_young((__vma)->vm_mm, __address, \
		__ptep); \
		__young; \
		})

		#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
		static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
		unsigned long addr, pte_t *ptep)
		{
		unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
		return __pte(old);
		}

		static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
		pte_t * ptep)
		{
		pte_update(mm, addr, ptep, ~0UL, 0, 0);
		}


		/* Set the dirty and/or accessed bits atomically in a linux PTE, this
		* function doesn't need to flush the hash entry
		*/
		static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry)
		{
		unsigned long bits = pte_val(entry) &
		(_PAGE_DIRTY \| _PAGE_ACCESSED \| _PAGE_RW \| _PAGE_EXEC);

		unsigned long old, tmp;

		__asm__ __volatile__(
		"1: ldarx %0,0,%4\n\
		andi. %1,%0,%6\n\
		bne- 1b \n\
		or %0,%3,%0\n\
		stdcx. %0,0,%4\n\
		bne- 1b"
		:"=&r" (old), "=&r" (tmp), "=m" (*ptep)
		:"r" (bits), "r" (ptep), "m" (*ptep), "i" (_PAGE_BUSY)
		:"cc");
		}

		#define __HAVE_ARCH_PTE_SAME
		#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HPTEFLAGS) == 0)

		static inline char get_hpte_slot_array(pmd_t pmdp)
		{
		/*
		* The hpte hindex is stored in the pgtable whose address is in the
		* second half of the PMD
		*
		* Order this load with the test for pmd_trans_huge in the caller
		*/
		smp_rmb();
		return (char *)(pmdp + PTRS_PER_PMD);


		}
		/*
		* The linux hugepage PMD now include the pmd entries followed by the address
		* to the stashed pgtable_t. The stashed pgtable_t contains the hpte bits.
		* [ 1 bit secondary \| 3 bit hidx \| 1 bit valid \| 000]. We use one byte per
		* each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and
		* with 4K HPTE we need 4096 entries. Both will fit in a 4K pgtable_t.
		*
		* The last three bits are intentionally left to zero. This memory location
		* are also used as normal page PTE pointers. So if we have any pointers
		* left around while we collapse a hugepage, we need to make sure
		* _PAGE_PRESENT bit of that is zero when we look at them
		*/
		static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index)
		{
		return (hpte_slot_array[index] >> 3) & 0x1;
		}

		static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array,
		int index)
		{
		return hpte_slot_array[index] >> 4;
		}

		static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array,
		unsigned int index, unsigned int hidx)
		{
		hpte_slot_array[index] = hidx << 4 \| 0x1 << 3;
		}

		#ifdef CONFIG_TRANSPARENT_HUGEPAGE
		/*
		*
		* For core kernel code by design pmd_trans_huge is never run on any hugetlbfs
		* page. The hugetlbfs page table walking and mangling paths are totally
		* separated form the core VM paths and they're differentiated by
		* VM_HUGETLB being set on vm_flags well before any pmd_trans_huge could run.
		*
		* pmd_trans_huge() is defined as false at build time if
		* CONFIG_TRANSPARENT_HUGEPAGE=n to optimize away code blocks at build
		* time in such case.
		*
		* For ppc64 we need to differntiate from explicit hugepages from THP, because
		* for THP we also track the subpage details at the pmd level. We don't do
		* that for explicit huge pages.
		*
		*/
		static inline int pmd_trans_huge(pmd_t pmd)
		{
		/*
		* leaf pte for huge page, bottom two bits != 00
		*/
		return (pmd_val(pmd) & 0x3) && (pmd_val(pmd) & _PAGE_THP_HUGE);
		}

		static inline int pmd_trans_splitting(pmd_t pmd)
		{
		if (pmd_trans_huge(pmd))
		return pmd_val(pmd) & _PAGE_SPLITTING;
		return 0;
		}

		#endif
		static inline int pmd_large(pmd_t pmd)
		{
		/*
		* leaf pte for huge page, bottom two bits != 00
		*/
		return ((pmd_val(pmd) & 0x3) != 0x0);
		}

		static inline pmd_t pmd_mknotpresent(pmd_t pmd)
		{
		return __pmd(pmd_val(pmd) & ~_PAGE_PRESENT);
		}

		static inline pmd_t pmd_mksplitting(pmd_t pmd)
		{
		return __pmd(pmd_val(pmd) \| _PAGE_SPLITTING);
		}

		#define __HAVE_ARCH_PMD_SAME
		static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
		{
		return (((pmd_val(pmd_a) ^ pmd_val(pmd_b)) & ~_PAGE_HPTEFLAGS) == 0);
		}

		static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
		unsigned long addr, pmd_t *pmdp)
		{
		unsigned long old;

		if ((pmd_val(*pmdp) & (_PAGE_ACCESSED \| _PAGE_HASHPTE)) == 0)
		return 0;
		old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0);
		return ((old & _PAGE_ACCESSED) != 0);
		}

		#define __HAVE_ARCH_PMDP_SET_WRPROTECT
		static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
		pmd_t *pmdp)
		{

		if ((pmd_val(*pmdp) & _PAGE_RW) == 0)
		return;

		pmd_hugepage_update(mm, addr, pmdp, _PAGE_RW, 0);
		}

		#ifdef CONFIG_TRANSPARENT_HUGEPAGE
		extern void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
		pmd_t *pmdp, unsigned long old_pmd);
		#else
		static inline void hpte_do_hugepage_flush(struct mm_struct *mm,
		unsigned long addr, pmd_t *pmdp,
		unsigned long old_pmd)
		{
		WARN(1, "%s called with THP disabled\n", __func__);
		}
		#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

		#endif /* !__ASSEMBLY__ */
		#endif /* __KERNEL__ */
		#endif /* _ASM_POWERPC_BOOK3S_64_HASH_H */

arch/powerpc/include/asm/book3s/64/pgtable.h

+3 −442

File changed.

Preview size limit exceeded, changes collapsed.

arch/powerpc/include/asm/pgtable.h

+0 −6

Original line number	Diff line number	Diff line
		@@ -18,12 +18,6 @@ struct mm_struct;
		#include <asm/pgtable-book3e.h>
		#endif /* !CONFIG_PPC_BOOK3S */

		/*
		* We save the slot number & secondary bit in the second half of the
		* PTE page. We use the 8 bytes per each pte entry.
		*/
		#define PTE_PAGE_HIDX_OFFSET (PTRS_PER_PTE * 8)

		#ifndef __ASSEMBLY__

		#include <asm/tlbflush.h>