Merge "ANDROID: mm/filemap: Fix missing put_page() for speculative page fault" (9e845bd9) · Commits · e / devices / android_kernel_fairphone_FP4

include/linux/mm.h

+6 −0

Original line number	Diff line number	Diff line
		@@ -1498,6 +1498,12 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
		#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
		static inline void vm_write_begin(struct vm_area_struct *vma)
		{
		/*
		* Isolated vma might be freed without exclusive mmap_lock but
		* speculative page fault handler still needs to know it was changed.
		*/
		if (!RB_EMPTY_NODE(&vma->vm_rb))
		WARN_ON_ONCE(!rwsem_is_locked(&(vma->vm_mm)->mmap_sem));
		/*
		* The reads never spins and preemption
		* disablement is not required.

mm/filemap.c

+49 −1

Original line number	Diff line number	Diff line
		@@ -2633,7 +2633,9 @@ static struct file do_async_mmap_readahead(struct vm_fault vmf,
		* it in the page cache, and handles the special cases reasonably without
		* having a lot of duplicated code.
		*
		* vma->vm_mm->mmap_sem must be held on entry (except FAULT_FLAG_SPECULATIVE).
		* If FAULT_FLAG_SPECULATIVE is set, this function runs with elevated vma
		* refcount and with mmap lock not held.
		* Otherwise, vma->vm_mm->mmap_sem must be held on entry.
		*
		* If our return value has VM_FAULT_RETRY set, it's because
		* lock_page_or_retry() returned 0.
		@@ -2658,6 +2660,52 @@ vm_fault_t filemap_fault(struct vm_fault *vmf)
		struct page *page;
		vm_fault_t ret = 0;

		if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
		page = find_get_page(mapping, offset);
		if (unlikely(!page))
		return VM_FAULT_RETRY;

		if (unlikely(PageReadahead(page)))
		goto page_put;

		if (!trylock_page(page))
		goto page_put;

		if (unlikely(compound_head(page)->mapping != mapping))
		goto page_unlock;
		VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page);
		if (unlikely(!PageUptodate(page)))
		goto page_unlock;

		max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
		if (unlikely(offset >= max_off))
		goto page_unlock;

		/*
		* Update readahead mmap_miss statistic.
		*
		* Note that we are not sure if finish_fault() will
		* manage to complete the transaction. If it fails,
		* we'll come back to filemap_fault() non-speculative
		* case which will update mmap_miss a second time.
		* This is not ideal, we would prefer to guarantee the
		* update will happen exactly once.
		*/
		if (!(vmf->vma->vm_flags & VM_RAND_READ) && ra->ra_pages) {
		unsigned int mmap_miss = READ_ONCE(ra->mmap_miss);
		if (mmap_miss)
		WRITE_ONCE(ra->mmap_miss, --mmap_miss);
		}

		vmf->page = page;
		return VM_FAULT_LOCKED;
		page_unlock:
		unlock_page(page);
		page_put:
		put_page(page);
		return VM_FAULT_RETRY;
		}

		max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
		if (unlikely(offset >= max_off))
		return VM_FAULT_SIGBUS;

mm/khugepaged.c

+2 −0

Original line number	Diff line number	Diff line
		@@ -1294,10 +1294,12 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
		*/
		if (down_write_trylock(&mm->mmap_sem)) {
		if (!khugepaged_test_exit(mm)) {
		vm_write_begin(vma);
		spinlock_t *ptl = pmd_lock(mm, pmd);
		/* assume page table is clear */
		_pmd = pmdp_collapse_flush(vma, addr, pmd);
		spin_unlock(ptl);
		vm_write_end(vma);
		mm_dec_nr_ptes(mm);
		pte_free(mm, pmd_pgtable(_pmd));
		}

mm/memory.c

+27 −19

Original line number	Diff line number	Diff line
		@@ -1553,7 +1553,6 @@ void unmap_page_range(struct mmu_gather *tlb,
		unsigned long next;

		BUG_ON(addr >= end);
		vm_write_begin(vma);
		tlb_start_vma(tlb, vma);
		pgd = pgd_offset(vma->vm_mm, addr);
		do {
		@@ -1563,7 +1562,6 @@ void unmap_page_range(struct mmu_gather *tlb,
		next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
		} while (pgd++, addr = next, addr != end);
		tlb_end_vma(tlb, vma);
		vm_write_end(vma);
		}


		@@ -3207,6 +3205,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
		int exclusive = 0;
		vm_fault_t ret;

		if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
		pte_unmap(vmf->pte);
		return VM_FAULT_RETRY;
		}

		ret = pte_unmap_same(vmf);
		if (ret) {
		/*
		@@ -3466,6 +3469,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
		if (vmf->vma_flags & VM_SHARED)
		return VM_FAULT_SIGBUS;

		/* Do not check unstable pmd, if it's changed will retry later */
		if (vmf->flags & FAULT_FLAG_SPECULATIVE)
		goto skip_pmd_checks;

		/*
		* Use pte_alloc() instead of pte_alloc_map(). We can't run
		* pte_offset_map() on pmds where a huge pmd might be created
		@@ -3483,6 +3490,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
		if (unlikely(pmd_trans_unstable(vmf->pmd)))
		return 0;

		skip_pmd_checks:
		/* Use the zero-page for reads */
		if (!(vmf->flags & FAULT_FLAG_WRITE) &&
		!mm_forbids_zeropage(vma->vm_mm)) {
		@@ -3587,6 +3595,10 @@ static vm_fault_t __do_fault(struct vm_fault *vmf)
		struct vm_area_struct *vma = vmf->vma;
		vm_fault_t ret;

		/* Do not check unstable pmd, if it's changed will retry later */
		if (vmf->flags & FAULT_FLAG_SPECULATIVE)
		goto skip_pmd_checks;

		/*
		* Preallocate pte before we take page_lock because this might lead to
		* deadlocks for memcg reclaim which waits for pages under writeback:
		@@ -3610,6 +3622,7 @@ static vm_fault_t __do_fault(struct vm_fault *vmf)
		smp_wmb(); /* See comment in __pte_alloc() */
		}

		skip_pmd_checks:
		ret = vma->vm_ops->fault(vmf);
		if (unlikely(ret & (VM_FAULT_ERROR \| VM_FAULT_NOPAGE \| VM_FAULT_RETRY \|
		VM_FAULT_DONE_COW)))
		@@ -3988,7 +4001,8 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf)
		end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1,
		start_pgoff + nr_pages - 1);

		if (pmd_none(*vmf->pmd)) {
		if (!(vmf->flags & FAULT_FLAG_SPECULATIVE) &&
		pmd_none(*vmf->pmd)) {
		vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm,
		vmf->address);
		if (!vmf->prealloc_pte)
		@@ -4356,16 +4370,11 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
		pte_t entry;
		int ret = 0;

		if (unlikely(pmd_none(*vmf->pmd))) {
		/*
		* In the case of the speculative page fault handler we abort
		* the speculative path immediately as the pmd is probably
		* in the way to be converted in a huge one. We will try
		* again holding the mmap_sem (which implies that the collapse
		* operation is done).
		*/
		/* Do not check unstable pmd, if it's changed will retry later */
		if (vmf->flags & FAULT_FLAG_SPECULATIVE)
		return VM_FAULT_RETRY;
		goto skip_pmd_checks;

		if (unlikely(pmd_none(*vmf->pmd))) {
		/*
		* Leave __pte_alloc() until later: because vm_ops->fault may
		* want to allocate huge page, and if we expose page table
		@@ -4373,8 +4382,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
		* concurrent faults and from rmap lookups.
		*/
		vmf->pte = NULL;
		} else if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) {
		/* See comment in pte_alloc_one_map() */
		} else {
		if (pmd_devmap_trans_unstable(vmf->pmd))
		return 0;
		/*
		@@ -4404,6 +4412,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
		}
		}

		skip_pmd_checks:
		if (!vmf->pte) {
		if (vma_is_anonymous(vmf->vma))
		return do_anonymous_page(vmf);
		@@ -4651,7 +4660,6 @@ int __handle_speculative_fault(struct mm_struct *mm, unsigned long address,
		pol = __get_vma_policy(vmf.vma, address);
		if (!pol)
		pol = get_task_policy(current);
		if (!pol)
		if (pol && pol->mode == MPOL_INTERLEAVE) {
		trace_spf_vma_notsup(_RET_IP_, vmf.vma, address);
		return VM_FAULT_RETRY;

mm/mempolicy.c

+0 −2

Original line number	Diff line number	Diff line
		@@ -599,11 +599,9 @@ unsigned long change_prot_numa(struct vm_area_struct *vma,
		{
		int nr_updated;

		vm_write_begin(vma);
		nr_updated = change_protection(vma, addr, end, PAGE_NONE, 0, 1);
		if (nr_updated)
		count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
		vm_write_end(vma);

		return nr_updated;
		}