ARM: LPAE: Page table maintenance for the 3-level format (da028779) · Commits · e / devices / android_kernel_oneplus_sm8150

arch/arm/include/asm/pgalloc.h

+24 −0

Original line number	Original line	Diff line number	Diff line
	@@ -25,6 +25,26 @@
	#define _PAGE_USER_TABLE (PMD_TYPE_TABLE \| PMD_BIT4 \| PMD_DOMAIN(DOMAIN_USER))		#define _PAGE_USER_TABLE (PMD_TYPE_TABLE \| PMD_BIT4 \| PMD_DOMAIN(DOMAIN_USER))
	#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE \| PMD_BIT4 \| PMD_DOMAIN(DOMAIN_KERNEL))		#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE \| PMD_BIT4 \| PMD_DOMAIN(DOMAIN_KERNEL))

			#ifdef CONFIG_ARM_LPAE

			static inline pmd_t pmd_alloc_one(struct mm_struct mm, unsigned long addr)
			{
			return (pmd_t *)get_zeroed_page(GFP_KERNEL \| __GFP_REPEAT);
			}

			static inline void pmd_free(struct mm_struct mm, pmd_t pmd)
			{
			BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
			free_page((unsigned long)pmd);
			}

			static inline void pud_populate(struct mm_struct mm, pud_t pud, pmd_t *pmd)
			{
			set_pud(pud, __pud(__pa(pmd) \| PMD_TYPE_TABLE));
			}

			#else /* !CONFIG_ARM_LPAE */

	/*		/*
	* Since we have only two-level page tables, these are trivial		* Since we have only two-level page tables, these are trivial
	*/		*/
	@@ -32,6 +52,8 @@
	#define pmd_free(mm, pmd) do { } while (0)		#define pmd_free(mm, pmd) do { } while (0)
	#define pud_populate(mm,pmd,pte) BUG()		#define pud_populate(mm,pmd,pte) BUG()

			#endif /* CONFIG_ARM_LPAE */

	extern pgd_t pgd_alloc(struct mm_struct mm);		extern pgd_t pgd_alloc(struct mm_struct mm);
	extern void pgd_free(struct mm_struct mm, pgd_t pgd);		extern void pgd_free(struct mm_struct mm, pgd_t pgd);

	@@ -109,7 +131,9 @@ static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte,
	{		{
	pmdval_t pmdval = (pte + PTE_HWTABLE_OFF) \| prot;		pmdval_t pmdval = (pte + PTE_HWTABLE_OFF) \| prot;
	pmdp[0] = __pmd(pmdval);		pmdp[0] = __pmd(pmdval);
			#ifndef CONFIG_ARM_LPAE
	pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));		pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
			#endif
	flush_pmd_entry(pmdp);		flush_pmd_entry(pmdp);
	}		}

arch/arm/include/asm/pgtable-3level.h

+53 −0

Original line number	Original line	Diff line number	Diff line
	@@ -99,4 +99,57 @@
	#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 3) << 2) /* normal inner write-back */		#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 3) << 2) /* normal inner write-back */
	#define L_PTE_MT_MASK (_AT(pteval_t, 7) << 2)		#define L_PTE_MT_MASK (_AT(pteval_t, 7) << 2)

			/*
			* Software PGD flags.
			*/
			#define L_PGD_SWAPPER (_AT(pgdval_t, 1) << 55) /* swapper_pg_dir entry */

			#ifndef __ASSEMBLY__

			#define pud_none(pud) (!pud_val(pud))
			#define pud_bad(pud) (!(pud_val(pud) & 2))
			#define pud_present(pud) (pud_val(pud))

			#define pud_clear(pudp) \
			do { \
			*pudp = __pud(0); \
			clean_pmd_entry(pudp); \
			} while (0)

			#define set_pud(pudp, pud) \
			do { \
			*pudp = pud; \
			flush_pmd_entry(pudp); \
			} while (0)

			static inline pmd_t *pud_page_vaddr(pud_t pud)
			{
			return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
			}

			/* Find an entry in the second-level page table.. */
			#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
			static inline pmd_t pmd_offset(pud_t pud, unsigned long addr)
			{
			return (pmd_t )pud_page_vaddr(pud) + pmd_index(addr);
			}

			#define pmd_bad(pmd) (!(pmd_val(pmd) & 2))

			#define copy_pmd(pmdpd,pmdps) \
			do { \
			pmdpd = pmdps; \
			flush_pmd_entry(pmdpd); \
			} while (0)

			#define pmd_clear(pmdp) \
			do { \
			*pmdp = __pmd(0); \
			clean_pmd_entry(pmdp); \
			} while (0)

			#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,__pte(pte_val(pte)\|(ext)))

			#endif /* __ASSEMBLY__ */

	#endif /* _ASM_PGTABLE_3LEVEL_H */		#endif /* _ASM_PGTABLE_3LEVEL_H */

arch/arm/include/asm/proc-fns.h

+21 −0

Original line number	Original line	Diff line number	Diff line
	@@ -65,7 +65,11 @@ extern struct processor {
	* Set a possibly extended PTE. Non-extended PTEs should		* Set a possibly extended PTE. Non-extended PTEs should
	* ignore 'ext'.		* ignore 'ext'.
	*/		*/
			#ifdef CONFIG_ARM_LPAE
			void (set_pte_ext)(pte_t ptep, pte_t pte);
			#else
	void (set_pte_ext)(pte_t ptep, pte_t pte, unsigned int ext);		void (set_pte_ext)(pte_t ptep, pte_t pte, unsigned int ext);
			#endif

	/* Suspend/resume */		/* Suspend/resume */
	unsigned int suspend_size;		unsigned int suspend_size;
	@@ -79,7 +83,11 @@ extern void cpu_proc_fin(void);
	extern int cpu_do_idle(void);		extern int cpu_do_idle(void);
	extern void cpu_dcache_clean_area(void *, int);		extern void cpu_dcache_clean_area(void *, int);
	extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);		extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
			#ifdef CONFIG_ARM_LPAE
			extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte);
			#else
	extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);		extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);
			#endif
	extern void cpu_reset(unsigned long addr) __attribute__((noreturn));		extern void cpu_reset(unsigned long addr) __attribute__((noreturn));

	/* These three are private to arch/arm/kernel/suspend.c */		/* These three are private to arch/arm/kernel/suspend.c */
	@@ -107,6 +115,18 @@ extern void cpu_resume(void);

	#define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm)		#define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm)

			#ifdef CONFIG_ARM_LPAE
			#define cpu_get_pgd() \
			({ \
			unsigned long pg, pg2; \
			__asm__("mrrc p15, 0, %0, %1, c2" \
			: "=r" (pg), "=r" (pg2) \
			: \
			: "cc"); \
			pg &= ~(PTRS_PER_PGD*sizeof(pgd_t)-1); \
			(pgd_t *)phys_to_virt(pg); \
			})
			#else
	#define cpu_get_pgd() \		#define cpu_get_pgd() \
	({ \		({ \
	unsigned long pg; \		unsigned long pg; \
	@@ -115,6 +135,7 @@ extern void cpu_resume(void);
	pg &= ~0x3fff; \		pg &= ~0x3fff; \
	(pgd_t *)phys_to_virt(pg); \		(pgd_t *)phys_to_virt(pg); \
	})		})
			#endif

	#endif		#endif

arch/arm/mm/ioremap.c

+5 −3

Original line number	Original line	Diff line number	Diff line
	@@ -64,7 +64,7 @@ void __check_kvm_seq(struct mm_struct *mm)
	} while (seq != init_mm.context.kvm_seq);		} while (seq != init_mm.context.kvm_seq);
	}		}

	#ifndef CONFIG_SMP		#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
	/*		/*
	* Section support is unsafe on SMP - If you iounmap and ioremap a region,		* Section support is unsafe on SMP - If you iounmap and ioremap a region,
	* the other CPUs will not see this change until their next context switch.		* the other CPUs will not see this change until their next context switch.
	@@ -202,11 +202,13 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
	unsigned long addr;		unsigned long addr;
	struct vm_struct * area;		struct vm_struct * area;

			#ifndef CONFIG_ARM_LPAE
	/*		/*
	* High mappings must be supersection aligned		* High mappings must be supersection aligned
	*/		*/
	if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK))		if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK))
	return NULL;		return NULL;
			#endif

	/*		/*
	* Don't allow RAM to be mapped - this causes problems with ARMv6+		* Don't allow RAM to be mapped - this causes problems with ARMv6+
	@@ -228,7 +230,7 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
	return NULL;		return NULL;
	addr = (unsigned long)area->addr;		addr = (unsigned long)area->addr;

	#ifndef CONFIG_SMP		#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
	if (DOMAIN_IO == 0 &&		if (DOMAIN_IO == 0 &&
	(((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) \|\|		(((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) \|\|
	cpu_is_xsc3()) && pfn >= 0x100000 &&		cpu_is_xsc3()) && pfn >= 0x100000 &&
	@@ -320,7 +322,7 @@ __arm_ioremap_exec(unsigned long phys_addr, size_t size, bool cached)
	void __iounmap(volatile void __iomem *io_addr)		void __iounmap(volatile void __iomem *io_addr)
	{		{
	void addr = (void )(PAGE_MASK & (unsigned long)io_addr);		void addr = (void )(PAGE_MASK & (unsigned long)io_addr);
	#ifndef CONFIG_SMP		#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
	struct vm_struct *p, tmp;		struct vm_struct *p, tmp;

	/*		/*

arch/arm/mm/pgd.c

+47 −4

Original line number	Original line	Diff line number	Diff line
	@@ -10,6 +10,7 @@
	#include <linux/mm.h>		#include <linux/mm.h>
	#include <linux/gfp.h>		#include <linux/gfp.h>
	#include <linux/highmem.h>		#include <linux/highmem.h>
			#include <linux/slab.h>

	#include <asm/pgalloc.h>		#include <asm/pgalloc.h>
	#include <asm/page.h>		#include <asm/page.h>
	@@ -17,6 +18,14 @@

	#include "mm.h"		#include "mm.h"

			#ifdef CONFIG_ARM_LPAE
			#define __pgd_alloc() kmalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL)
			#define __pgd_free(pgd) kfree(pgd)
			#else
			#define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL, 2)
			#define __pgd_free(pgd) free_pages((unsigned long)pgd, 2)
			#endif

	/*		/*
	* need to get a 16k page for level 1		* need to get a 16k page for level 1
	*/		*/
	@@ -27,7 +36,7 @@ pgd_t pgd_alloc(struct mm_struct mm)
	pmd_t new_pmd, init_pmd;		pmd_t new_pmd, init_pmd;
	pte_t new_pte, init_pte;		pte_t new_pte, init_pte;

	new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2);		new_pgd = __pgd_alloc();
	if (!new_pgd)		if (!new_pgd)
	goto no_pgd;		goto no_pgd;

	@@ -42,10 +51,25 @@ pgd_t pgd_alloc(struct mm_struct mm)

	clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));		clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));

			#ifdef CONFIG_ARM_LPAE
			/*
			* Allocate PMD table for modules and pkmap mappings.
			*/
			new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR),
			MODULES_VADDR);
			if (!new_pud)
			goto no_pud;

			new_pmd = pmd_alloc(mm, new_pud, 0);
			if (!new_pmd)
			goto no_pmd;
			#endif

	if (!vectors_high()) {		if (!vectors_high()) {
	/*		/*
	* On ARM, first page must always be allocated since it		* On ARM, first page must always be allocated since it
	* contains the machine vectors.		* contains the machine vectors. The vectors are always high
			* with LPAE.
	*/		*/
	new_pud = pud_alloc(mm, new_pgd, 0);		new_pud = pud_alloc(mm, new_pgd, 0);
	if (!new_pud)		if (!new_pud)
	@@ -74,7 +98,7 @@ pgd_t pgd_alloc(struct mm_struct mm)
	no_pmd:		no_pmd:
	pud_free(mm, new_pud);		pud_free(mm, new_pud);
	no_pud:		no_pud:
	free_pages((unsigned long)new_pgd, 2);		__pgd_free(new_pgd);
	no_pgd:		no_pgd:
	return NULL;		return NULL;
	}		}
	@@ -111,5 +135,24 @@ void pgd_free(struct mm_struct mm, pgd_t pgd_base)
	pgd_clear(pgd);		pgd_clear(pgd);
	pud_free(mm, pud);		pud_free(mm, pud);
	no_pgd:		no_pgd:
	free_pages((unsigned long) pgd_base, 2);		#ifdef CONFIG_ARM_LPAE
			/*
			* Free modules/pkmap or identity pmd tables.
			*/
			for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) {
			if (pgd_none_or_clear_bad(pgd))
			continue;
			if (pgd_val(*pgd) & L_PGD_SWAPPER)
			continue;
			pud = pud_offset(pgd, 0);
			if (pud_none_or_clear_bad(pud))
			continue;
			pmd = pmd_offset(pud, 0);
			pud_clear(pud);
			pmd_free(mm, pmd);
			pgd_clear(pgd);
			pud_free(mm, pud);
			}
			#endif
			__pgd_free(pgd_base);
	}		}