Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c3e62fff authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Vinayak Menon
Browse files

mm: prepare for FAULT_FLAG_SPECULATIVE 



When speculating faults (without holding mmap_sem) we need to validate
that the vma against which we loaded pages is still valid when we're
ready to install the new PTE.

Therefore, replace the pte_offset_map_lock() calls that (re)take the
PTL with pte_map_lock() which can fail in case we find the VMA changed
since we started the fault.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>

[Port to 4.12 kernel]
[Remove the comment about the fault_env structure which has been
 implemented as the vm_fault structure in the kernel]
[move pte_map_lock()'s definition upper in the file]
[move the define of FAULT_FLAG_SPECULATIVE later in the series]
[review error path in do_swap_page(), do_anonymous_page() and
 wp_page_copy()]
Signed-off-by: default avatarLaurent Dufour <ldufour@linux.vnet.ibm.com>
Change-Id: Id6dfae130fbfdd4bb92aa6415d6f1db7ef833266
Patch-mainline: linux-mm @ Tue, 17 Apr 2018 16:33:10
[vinmenon@codeaurora.org: fix merge conflicts]
Signed-off-by: default avatarVinayak Menon <vinmenon@codeaurora.org>
parent 643637bb

mm/memory.c

+61 −29
Original line number Original line Diff line number Diff line
@@ -1961,6 +1961,14 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr, 
}

}

EXPORT_SYMBOL_GPL(apply_to_page_range);

EXPORT_SYMBOL_GPL(apply_to_page_range);





static inline bool pte_map_lock(struct mm_struct *mm,

			struct fault_env *fe)

{

	fe->pte = pte_offset_map_lock(mm, fe->pmd,

				       fe->address, &fe->ptl);

	return true;

}



/*

/*

 * handle_pte_fault chooses page fault handler according to an entry which was

 * handle_pte_fault chooses page fault handler according to an entry which was

 * read non-atomically.  Before making any commitment, on those architectures

 * read non-atomically.  Before making any commitment, on those architectures
@@ -2145,24 +2153,25 @@ static int wp_page_copy(struct fault_env *fe, pte_t orig_pte, 
	const unsigned long mmun_start = fe->address & PAGE_MASK;

	const unsigned long mmun_start = fe->address & PAGE_MASK;

	const unsigned long mmun_end = mmun_start + PAGE_SIZE;

	const unsigned long mmun_end = mmun_start + PAGE_SIZE;

	struct mem_cgroup *memcg;

	struct mem_cgroup *memcg;

	int ret = VM_FAULT_OOM;





	if (unlikely(anon_vma_prepare(vma)))

	if (unlikely(anon_vma_prepare(vma)))

		goto oom;

		goto out;





	if (is_zero_pfn(pte_pfn(orig_pte))) {

	if (is_zero_pfn(pte_pfn(orig_pte))) {

		new_page = alloc_zeroed_user_highpage_movable(vma, fe->address);

		new_page = alloc_zeroed_user_highpage_movable(vma, fe->address);

		if (!new_page)

		if (!new_page)

			goto oom;

			goto out;

	} else {

	} else {

		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,

		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,

				fe->address);

				fe->address);

		if (!new_page)

		if (!new_page)

			goto oom;

			goto out;

		cow_user_page(new_page, old_page, fe->address, vma);

		cow_user_page(new_page, old_page, fe->address, vma);

	}

	}





	if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg, false))

	if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg, false))

		goto oom_free_new;

		goto out_free_new;





	__SetPageUptodate(new_page);

	__SetPageUptodate(new_page);
@@ -2171,7 +2180,10 @@ static int wp_page_copy(struct fault_env *fe, pte_t orig_pte, 
	/*

	/*

	 * Re-check the pte - we dropped the lock

	 * Re-check the pte - we dropped the lock

	 */

	 */

	fe->pte = pte_offset_map_lock(mm, fe->pmd, fe->address, &fe->ptl);

	if (!pte_map_lock(mm, fe)) {

		ret = VM_FAULT_RETRY;

		goto out_uncharge;

	}

	if (likely(pte_same(*fe->pte, orig_pte))) {

	if (likely(pte_same(*fe->pte, orig_pte))) {

		if (old_page) {

		if (old_page) {

			if (!PageAnon(old_page)) {

			if (!PageAnon(old_page)) {
@@ -2254,12 +2266,14 @@ static int wp_page_copy(struct fault_env *fe, pte_t orig_pte, 
		put_page(old_page);

		put_page(old_page);

	}

	}

	return page_copied ? VM_FAULT_WRITE : 0;

	return page_copied ? VM_FAULT_WRITE : 0;

oom_free_new:

out_uncharge:

	mem_cgroup_cancel_charge(new_page, memcg, false);

out_free_new:

	put_page(new_page);

	put_page(new_page);

oom:

out:

	if (old_page)

	if (old_page)

		put_page(old_page);

		put_page(old_page);

	return VM_FAULT_OOM;

	return ret;

}

}





/*

/*
@@ -2284,8 +2298,8 @@ static int wp_pfn_shared(struct fault_env *fe, pte_t orig_pte) 
		ret = vma->vm_ops->pfn_mkwrite(vma, &vmf);

		ret = vma->vm_ops->pfn_mkwrite(vma, &vmf);

		if (ret & VM_FAULT_ERROR)

		if (ret & VM_FAULT_ERROR)

			return ret;

			return ret;

		fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,

		if (!pte_map_lock(vma->vm_mm, fe))

				&fe->ptl);

			return VM_FAULT_RETRY;

		/*

		/*

		 * We might have raced with another page fault while we

		 * We might have raced with another page fault while we

		 * released the pte_offset_map_lock.

		 * released the pte_offset_map_lock.
@@ -2388,8 +2402,11 @@ static int do_wp_page(struct fault_env *fe, pte_t orig_pte) 
			get_page(old_page);

			get_page(old_page);

			pte_unmap_unlock(fe->pte, fe->ptl);

			pte_unmap_unlock(fe->pte, fe->ptl);

			lock_page(old_page);

			lock_page(old_page);

			fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd,

			if (!pte_map_lock(vma->vm_mm, fe)) {

					fe->address, &fe->ptl);

				unlock_page(old_page);

				put_page(old_page);

				return VM_FAULT_RETRY;

			}

			if (!pte_same(*fe->pte, orig_pte)) {

			if (!pte_same(*fe->pte, orig_pte)) {

				unlock_page(old_page);

				unlock_page(old_page);

				pte_unmap_unlock(fe->pte, fe->ptl);

				pte_unmap_unlock(fe->pte, fe->ptl);
@@ -2545,11 +2562,16 @@ int do_swap_page(struct fault_env *fe, pte_t orig_pte) 
					GFP_HIGHUSER_MOVABLE, vma, fe->address);

					GFP_HIGHUSER_MOVABLE, vma, fe->address);

		if (!page) {

		if (!page) {

			/*

			/*

			 * Back out if somebody else faulted in this pte

			 * Back out if the VMA has changed in our back during

			 * while we released the pte lock.

			 * a speculative page fault or if somebody else

			 * faulted in this pte while we released the pte lock.

			 */

			 */

			fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd,

			if (!pte_map_lock(vma->vm_mm, fe)) {

					fe->address, &fe->ptl);

				delayacct_clear_flag(DELAYACCT_PF_SWAPIN);

				ret = VM_FAULT_RETRY;

				goto out;

			}



			if (likely(pte_same(*fe->pte, orig_pte)))

			if (likely(pte_same(*fe->pte, orig_pte)))

				ret = VM_FAULT_OOM;

				ret = VM_FAULT_OOM;

			delayacct_clear_flag(DELAYACCT_PF_SWAPIN);

			delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
@@ -2603,10 +2625,13 @@ int do_swap_page(struct fault_env *fe, pte_t orig_pte) 
	}

	}





	/*

	/*

	 * Back out if somebody else already faulted in this pte.

	 * Back out if the VMA has changed in our back during a speculative

	 * page fault or if somebody else already faulted in this pte.

	 */

	 */

	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,

	if (!pte_map_lock(vma->vm_mm, fe)) {

			&fe->ptl);

		ret = VM_FAULT_RETRY;

		goto out_cancel_cgroup;

	}

	if (unlikely(!pte_same(*fe->pte, orig_pte)))

	if (unlikely(!pte_same(*fe->pte, orig_pte)))

		goto out_nomap;

		goto out_nomap;
@@ -2680,8 +2705,9 @@ int do_swap_page(struct fault_env *fe, pte_t orig_pte) 
out:

out:

	return ret;

	return ret;

out_nomap:

out_nomap:

	mem_cgroup_cancel_charge(page, memcg, false);

	pte_unmap_unlock(fe->pte, fe->ptl);

	pte_unmap_unlock(fe->pte, fe->ptl);

out_cancel_cgroup:

	mem_cgroup_cancel_charge(page, memcg, false);

out_page:

out_page:

	unlock_page(page);

	unlock_page(page);

out_release:

out_release:
@@ -2703,6 +2729,7 @@ static int do_anonymous_page(struct fault_env *fe) 
	struct vm_area_struct *vma = fe->vma;

	struct vm_area_struct *vma = fe->vma;

	struct mem_cgroup *memcg;

	struct mem_cgroup *memcg;

	struct page *page;

	struct page *page;

	int ret = 0;

	pte_t entry;

	pte_t entry;





	/* File mapping without ->vm_ops ? */

	/* File mapping without ->vm_ops ? */
@@ -2731,8 +2758,8 @@ static int do_anonymous_page(struct fault_env *fe) 
			!mm_forbids_zeropage(vma->vm_mm)) {

			!mm_forbids_zeropage(vma->vm_mm)) {

		entry = pte_mkspecial(pfn_pte(my_zero_pfn(fe->address),

		entry = pte_mkspecial(pfn_pte(my_zero_pfn(fe->address),

						vma->vm_page_prot));

						vma->vm_page_prot));

		fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,

		if (!pte_map_lock(vma->vm_mm, fe))

				&fe->ptl);

			return VM_FAULT_RETRY;

		if (!pte_none(*fe->pte))

		if (!pte_none(*fe->pte))

			goto unlock;

			goto unlock;

		/* Deliver the page fault to userland, check inside PT lock */

		/* Deliver the page fault to userland, check inside PT lock */
@@ -2764,10 +2791,12 @@ static int do_anonymous_page(struct fault_env *fe) 
	if (vma->vm_flags & VM_WRITE)

	if (vma->vm_flags & VM_WRITE)

		entry = pte_mkwrite(pte_mkdirty(entry));

		entry = pte_mkwrite(pte_mkdirty(entry));





	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,

	if (!pte_map_lock(vma->vm_mm, fe)) {

			&fe->ptl);

		ret = VM_FAULT_RETRY;

	if (!pte_none(*fe->pte))

		goto release;

		goto release;

	}

	if (!pte_none(*fe->pte))

		goto unlock_and_release;





	/* Deliver the page fault to userland, check inside PT lock */

	/* Deliver the page fault to userland, check inside PT lock */

	if (userfaultfd_missing(vma)) {

	if (userfaultfd_missing(vma)) {
@@ -2788,11 +2817,13 @@ static int do_anonymous_page(struct fault_env *fe) 
	update_mmu_cache(vma, fe->address, fe->pte);

	update_mmu_cache(vma, fe->address, fe->pte);

unlock:

unlock:

	pte_unmap_unlock(fe->pte, fe->ptl);

	pte_unmap_unlock(fe->pte, fe->ptl);

	return 0;

	return ret;

unlock_and_release:

	pte_unmap_unlock(fe->pte, fe->ptl);

release:

release:

	mem_cgroup_cancel_charge(page, memcg, false);

	mem_cgroup_cancel_charge(page, memcg, false);

	put_page(page);

	put_page(page);

	goto unlock;

	return ret;

oom_free_page:

oom_free_page:

	put_page(page);

	put_page(page);

oom:

oom:
@@ -2897,8 +2928,9 @@ static int pte_alloc_one_map(struct fault_env *fe) 
	 * pte_none() under vmf->ptl protection when we return to

	 * pte_none() under vmf->ptl protection when we return to

	 * alloc_set_pte().

	 * alloc_set_pte().

	 */

	 */

	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,

	if (!pte_map_lock(vma->vm_mm, fe))

			&fe->ptl);

		return VM_FAULT_RETRY;



	return 0;

	return 0;

}

}