Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 544f86ff authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Vinayak Menon
Browse files

mm: prepare for FAULT_FLAG_SPECULATIVE 



When speculating faults (without holding mmap_sem) we need to validate
that the vma against which we loaded pages is still valid when we're
ready to install the new PTE.

Therefore, replace the pte_offset_map_lock() calls that (re)take the
PTL with pte_map_lock() which can fail in case we find the VMA changed
since we started the fault.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>

[Port to 4.12 kernel]
[Remove the comment about the fault_env structure which has been
 implemented as the vm_fault structure in the kernel]
[move pte_map_lock()'s definition upper in the file]
[move the define of FAULT_FLAG_SPECULATIVE later in the series]
[review error path in do_swap_page(), do_anonymous_page() and
 wp_page_copy()]
Signed-off-by: default avatarLaurent Dufour <ldufour@linux.vnet.ibm.com>
Change-Id: Id6dfae130fbfdd4bb92aa6415d6f1db7ef833266
[vinmenon@codeaurora.org: fix trivial merge conflicts]
Patch-mainline: linux-mm @ Tue, 17 Apr 2018 16:33:10
Signed-off-by: default avatarVinayak Menon <vinmenon@codeaurora.org>
parent c679e340

mm/memory.c

+59 −29
Original line number Diff line number Diff line
@@ -2270,6 +2270,13 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr, 
}

EXPORT_SYMBOL_GPL(apply_to_page_range);



static inline bool pte_map_lock(struct vm_fault *vmf)

{

	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,

				       vmf->address, &vmf->ptl);

	return true;

}



/*

 * handle_pte_fault chooses page fault handler according to an entry which was

 * read non-atomically.  Before making any commitment, on those architectures
@@ -2459,25 +2466,26 @@ static int wp_page_copy(struct vm_fault *vmf) 
	const unsigned long mmun_start = vmf->address & PAGE_MASK;

	const unsigned long mmun_end = mmun_start + PAGE_SIZE;

	struct mem_cgroup *memcg;

	int ret = VM_FAULT_OOM;



	if (unlikely(anon_vma_prepare(vma)))

		goto oom;

		goto out;



	if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {

		new_page = alloc_zeroed_user_highpage_movable(vma,

							      vmf->address);

		if (!new_page)

			goto oom;

			goto out;

	} else {

		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,

				vmf->address);

		if (!new_page)

			goto oom;

			goto out;

		cow_user_page(new_page, old_page, vmf->address, vma);

	}



	if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg, false))

		goto oom_free_new;

		goto out_free_new;



	__SetPageUptodate(new_page);
@@ -2486,7 +2494,10 @@ static int wp_page_copy(struct vm_fault *vmf) 
	/*

	 * Re-check the pte - we dropped the lock

	 */

	vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);

	if (!pte_map_lock(vmf)) {

		ret = VM_FAULT_RETRY;

		goto out_uncharge;

	}

	if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {

		if (old_page) {

			if (!PageAnon(old_page)) {
@@ -2569,12 +2580,14 @@ static int wp_page_copy(struct vm_fault *vmf) 
		put_page(old_page);

	}

	return page_copied ? VM_FAULT_WRITE : 0;

oom_free_new:

out_uncharge:

	mem_cgroup_cancel_charge(new_page, memcg, false);

out_free_new:

	put_page(new_page);

oom:

out:

	if (old_page)

		put_page(old_page);

	return VM_FAULT_OOM;

	return ret;

}



/**
@@ -2595,8 +2608,8 @@ static int wp_page_copy(struct vm_fault *vmf) 
int finish_mkwrite_fault(struct vm_fault *vmf)

{

	WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));

	vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,

				       &vmf->ptl);

	if (!pte_map_lock(vmf))

		return VM_FAULT_RETRY;

	/*

	 * We might have raced with another page fault while we released the

	 * pte_offset_map_lock.
@@ -2714,8 +2727,11 @@ static int do_wp_page(struct vm_fault *vmf) 
			get_page(vmf->page);

			pte_unmap_unlock(vmf->pte, vmf->ptl);

			lock_page(vmf->page);

			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,

					vmf->address, &vmf->ptl);

			if (!pte_map_lock(vmf)) {

				unlock_page(vmf->page);

				put_page(vmf->page);

				return VM_FAULT_RETRY;

			}

			if (!pte_same(*vmf->pte, vmf->orig_pte)) {

				unlock_page(vmf->page);

				pte_unmap_unlock(vmf->pte, vmf->ptl);
@@ -2903,11 +2919,16 @@ int do_swap_page(struct vm_fault *vmf) 


		if (!page) {

			/*

			 * Back out if somebody else faulted in this pte

			 * while we released the pte lock.

			 * Back out if the VMA has changed in our back during

			 * a speculative page fault or if somebody else

			 * faulted in this pte while we released the pte lock.

			 */

			vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,

					vmf->address, &vmf->ptl);

			if (!pte_map_lock(vmf)) {

				delayacct_clear_flag(DELAYACCT_PF_SWAPIN);

				ret = VM_FAULT_RETRY;

				goto out;

			}



			if (likely(pte_same(*vmf->pte, vmf->orig_pte)))

				ret = VM_FAULT_OOM;

			delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
@@ -2960,10 +2981,13 @@ int do_swap_page(struct vm_fault *vmf) 
	}



	/*

	 * Back out if somebody else already faulted in this pte.

	 * Back out if the VMA has changed in our back during a speculative

	 * page fault or if somebody else already faulted in this pte.

	 */

	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,

			&vmf->ptl);

	if (!pte_map_lock(vmf)) {

		ret = VM_FAULT_RETRY;

		goto out_cancel_cgroup;

	}

	if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))

		goto out_nomap;
@@ -3040,8 +3064,9 @@ int do_swap_page(struct vm_fault *vmf) 
out:

	return ret;

out_nomap:

	mem_cgroup_cancel_charge(page, memcg, false);

	pte_unmap_unlock(vmf->pte, vmf->ptl);

out_cancel_cgroup:

	mem_cgroup_cancel_charge(page, memcg, false);

out_page:

	unlock_page(page);

out_release:
@@ -3092,8 +3117,8 @@ static int do_anonymous_page(struct vm_fault *vmf) 
			!mm_forbids_zeropage(vma->vm_mm)) {

		entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),

						vma->vm_page_prot));

		vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,

				vmf->address, &vmf->ptl);

		if (!pte_map_lock(vmf))

			return VM_FAULT_RETRY;

		if (!pte_none(*vmf->pte))

			goto unlock;

		ret = check_stable_address_space(vma->vm_mm);
@@ -3128,14 +3153,16 @@ static int do_anonymous_page(struct vm_fault *vmf) 
	if (vma->vm_flags & VM_WRITE)

		entry = pte_mkwrite(pte_mkdirty(entry));



	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,

			&vmf->ptl);

	if (!pte_none(*vmf->pte))

	if (!pte_map_lock(vmf)) {

		ret = VM_FAULT_RETRY;

		goto release;

	}

	if (!pte_none(*vmf->pte))

		goto unlock_and_release;



	ret = check_stable_address_space(vma->vm_mm);

	if (ret)

		goto release;

		goto unlock_and_release;



	/* Deliver the page fault to userland, check inside PT lock */

	if (userfaultfd_missing(vma)) {
@@ -3157,10 +3184,12 @@ static int do_anonymous_page(struct vm_fault *vmf) 
unlock:

	pte_unmap_unlock(vmf->pte, vmf->ptl);

	return ret;

unlock_and_release:

	pte_unmap_unlock(vmf->pte, vmf->ptl);

release:

	mem_cgroup_cancel_charge(page, memcg, false);

	put_page(page);

	goto unlock;

	return ret;

oom_free_page:

	put_page(page);

oom:
@@ -3253,8 +3282,9 @@ static int pte_alloc_one_map(struct vm_fault *vmf) 
	 * pte_none() under vmf->ptl protection when we return to

	 * alloc_set_pte().

	 */

	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,

			&vmf->ptl);

	if (!pte_map_lock(vmf))

		return VM_FAULT_RETRY;



	return 0;

}