Merge "arm64/mm: add speculative page fault" (4330dfa8) · Commits · e / devices / android_kernel_fairphone_FP4

arch/arm64/Kconfig

+1 −0

Original line number	Diff line number	Diff line
		@@ -166,6 +166,7 @@ config ARM64
		select SWIOTLB
		select SYSCTL_EXCEPTION_TRACE
		select THREAD_INFO_IN_TASK
		select ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT
		help
		ARM 64-bit (AArch64) Linux support.

arch/arm64/mm/fault.c

+23 −4

Original line number	Diff line number	Diff line
		@@ -380,14 +380,12 @@ static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *re
		#define VM_FAULT_BADMAP 0x010000
		#define VM_FAULT_BADACCESS 0x020000

		static vm_fault_t __do_page_fault(struct mm_struct *mm, unsigned long addr,
		static int __do_page_fault(struct vm_area_struct *vma, unsigned long addr,
		unsigned int mm_flags, unsigned long vm_flags,
		struct task_struct *tsk)
		{
		struct vm_area_struct *vma;
		vm_fault_t fault;

		vma = find_vma(mm, addr);
		fault = VM_FAULT_BADMAP;
		if (unlikely(!vma))
		goto out;
		@@ -431,6 +429,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
		vm_fault_t fault, major = 0;
		unsigned long vm_flags = VM_READ \| VM_WRITE;
		unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY \| FAULT_FLAG_KILLABLE;
		struct vm_area_struct *vma = NULL;

		if (notify_page_fault(regs, esr))
		return 0;
		@@ -472,6 +471,14 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,

		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);

		/*
		* let's try a speculative page fault without grabbing the
		* mmap_sem.
		*/
		fault = handle_speculative_fault(mm, addr, mm_flags, &vma);
		if (fault != VM_FAULT_RETRY)
		goto done;

		/*
		* As per x86, we may deadlock here. However, since the kernel only
		* validly references user space from well defined areas of the code,
		@@ -494,7 +501,10 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
		#endif
		}

		fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
		if (!vma \|\| !can_reuse_spf_vma(vma, addr))
		vma = find_vma(mm, addr);

		fault = __do_page_fault(vma, addr, mm_flags, vm_flags, tsk);
		major \|= fault & VM_FAULT_MAJOR;

		if (fault & VM_FAULT_RETRY) {
		@@ -517,11 +527,20 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
		if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
		mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
		mm_flags \|= FAULT_FLAG_TRIED;

		/*
		* Do not try to reuse this vma and fetch it
		* again since we will release the mmap_sem.
		*/
		vma = NULL;

		goto retry;
		}
		}
		up_read(&mm->mmap_sem);

		done:

		/*
		* Handle the "normal" (no error) case first.
		*/

include/linux/hugetlb_inline.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -8,7 +8,7 @@

		static inline bool is_vm_hugetlb_page(struct vm_area_struct *vma)
		{
		return !!(vma->vm_flags & VM_HUGETLB);
		return !!(READ_ONCE(vma->vm_flags) & VM_HUGETLB);
		}

		#else

include/linux/mm.h

+44 −0

Original line number	Diff line number	Diff line
		@@ -319,6 +319,8 @@ extern pgprot_t protection_map[16];
		#define FAULT_FLAG_REMOTE 0x80 /* faulting for non current tsk/mm */
		#define FAULT_FLAG_INSTRUCTION 0x100 /* The fault was during an instruction fetch */
		#define FAULT_FLAG_PREFAULT_OLD 0x400 /* Make faultaround ptes old */
		/* Speculative fault, not holding mmap_sem */
		#define FAULT_FLAG_SPECULATIVE 0x200

		#define FAULT_FLAG_TRACE \
		{ FAULT_FLAG_WRITE, "WRITE" }, \
		@@ -347,6 +349,10 @@ struct vm_fault {
		gfp_t gfp_mask; /* gfp mask to be used for allocations */
		pgoff_t pgoff; /* Logical page offset based on vma */
		unsigned long address; /* Faulting virtual address */
		#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
		unsigned int sequence;
		pmd_t orig_pmd; /* value of PMD at the time of fault */
		#endif
		pmd_t pmd; / Pointer to pmd entry matching
		* the 'address' */
		pud_t pud; / Pointer to pud entry matching
		@@ -464,6 +470,7 @@ static inline void INIT_VMA(struct vm_area_struct *vma)
		INIT_LIST_HEAD(&vma->anon_vma_chain);
		#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
		seqcount_init(&vma->vm_sequence);
		atomic_set(&vma->vm_ref_count, 1);
		#endif
		}

		@@ -1474,6 +1481,43 @@ int invalidate_inode_page(struct page *page);
		#ifdef CONFIG_MMU
		extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
		unsigned long address, unsigned int flags);

		#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
		extern int __handle_speculative_fault(struct mm_struct *mm,
		unsigned long address,
		unsigned int flags,
		struct vm_area_struct **vma);
		static inline int handle_speculative_fault(struct mm_struct *mm,
		unsigned long address,
		unsigned int flags,
		struct vm_area_struct **vma)
		{
		/*
		* Try speculative page fault for multithreaded user space task only.
		*/
		if (!(flags & FAULT_FLAG_USER) \|\| atomic_read(&mm->mm_users) == 1) {
		*vma = NULL;
		return VM_FAULT_RETRY;
		}
		return __handle_speculative_fault(mm, address, flags, vma);
		}
		extern bool can_reuse_spf_vma(struct vm_area_struct *vma,
		unsigned long address);
		#else
		static inline int handle_speculative_fault(struct mm_struct *mm,
		unsigned long address,
		unsigned int flags,
		struct vm_area_struct **vma)
		{
		return VM_FAULT_RETRY;
		}
		static inline bool can_reuse_spf_vma(struct vm_area_struct *vma,
		unsigned long address)
		{
		return false;
		}
		#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */

		extern int fixup_user_fault(struct task_struct tsk, struct mm_struct mm,
		unsigned long address, unsigned int fault_flags,
		bool *unlocked);

include/linux/mm_types.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -332,6 +332,7 @@ struct vm_area_struct {
		struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
		#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
		seqcount_t vm_sequence;
		atomic_t vm_ref_count; /* see vma_get(), vma_put() */
		#endif
		} __randomize_layout;

		@@ -352,6 +353,9 @@ struct mm_struct {
		struct vm_area_struct mmap; / list of VMAs */
		struct rb_root mm_rb;
		u64 vmacache_seqnum; /* per-thread vmacache */
		#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
		rwlock_t mm_rb_lock;
		#endif
		#ifdef CONFIG_MMU
		unsigned long (get_unmapped_area) (struct file filp,
		unsigned long addr, unsigned long len,