Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 53f5c3f4 authored by Jérôme Glisse's avatar Jérôme Glisse Committed by Linus Torvalds
Browse files

mm/hmm: factor out pte and pmd handling to simplify hmm_vma_walk_pmd() 

No functional change, just create one function to handle pmd and one to
handle pte (hmm_vma_handle_pmd() and hmm_vma_handle_pte()).

Link: http://lkml.kernel.org/r/20180323005527.758-14-jglisse@redhat.com


Signed-off-by: default avatarJérôme Glisse <jglisse@redhat.com>
Reviewed-by: default avatarJohn Hubbard <jhubbard@nvidia.com>
Cc: Evgeny Baskakov <ebaskakov@nvidia.com>
Cc: Ralph Campbell <rcampbell@nvidia.com>
Cc: Mark Hairgrove <mhairgrove@nvidia.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 33cd47dc

mm/hmm.c

+102 −72
Original line number Diff line number Diff line
@@ -375,74 +375,45 @@ static int hmm_vma_walk_hole(unsigned long addr, 
	return hmm_vma_walk->fault ? -EAGAIN : 0;

}



static int hmm_vma_walk_pmd(pmd_t *pmdp,

			    unsigned long start,

static int hmm_vma_handle_pmd(struct mm_walk *walk,

			      unsigned long addr,

			      unsigned long end,

			    struct mm_walk *walk)

			      uint64_t *pfns,

			      pmd_t pmd)

{

	struct hmm_vma_walk *hmm_vma_walk = walk->private;

	struct hmm_range *range = hmm_vma_walk->range;

	struct vm_area_struct *vma = walk->vma;

	uint64_t *pfns = range->pfns;

	unsigned long addr = start, i;

	bool write_fault;

	pte_t *ptep;



	i = (addr - range->start) >> PAGE_SHIFT;

	write_fault = hmm_vma_walk->fault & hmm_vma_walk->write;



again:

	if (pmd_none(*pmdp))

		return hmm_vma_walk_hole(start, end, walk);



	if (pmd_huge(*pmdp) && vma->vm_flags & VM_HUGETLB)

		return hmm_pfns_bad(start, end, walk);



	if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {

		unsigned long pfn;

	unsigned long pfn, i;

	uint64_t flag = 0;

		pmd_t pmd;



		/*

		 * No need to take pmd_lock here, even if some other threads

		 * is splitting the huge pmd we will get that event through

		 * mmu_notifier callback.

		 *

		 * So just read pmd value and check again its a transparent

		 * huge or device mapping one and compute corresponding pfn

		 * values.

		 */

		pmd = pmd_read_atomic(pmdp);

		barrier();

		if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))

			goto again;

	if (pmd_protnone(pmd))

			return hmm_vma_walk_hole(start, end, walk);

		return hmm_vma_walk_hole(addr, end, walk);



		if (write_fault && !pmd_write(pmd))

			return hmm_vma_walk_hole(start, end, walk);

	if ((hmm_vma_walk->fault & hmm_vma_walk->write) && !pmd_write(pmd))

		return hmm_vma_walk_hole(addr, end, walk);



	pfn = pmd_pfn(pmd) + pte_index(addr);

	flag |= pmd_write(pmd) ? HMM_PFN_WRITE : 0;

		for (; addr < end; addr += PAGE_SIZE, i++, pfn++)

	for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)

		pfns[i] = hmm_pfn_from_pfn(pfn) | flag;

	hmm_vma_walk->last = end;

	return 0;

}



	if (pmd_bad(*pmdp))

		return hmm_pfns_bad(start, end, walk);



	ptep = pte_offset_map(pmdp, addr);

	for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {

static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,

			      unsigned long end, pmd_t *pmdp, pte_t *ptep,

			      uint64_t *pfn)

{

	struct hmm_vma_walk *hmm_vma_walk = walk->private;

	struct vm_area_struct *vma = walk->vma;

	pte_t pte = *ptep;



		pfns[i] = 0;

	*pfn = 0;



	if (pte_none(pte)) {

			pfns[i] = 0;

		*pfn = 0;

		if (hmm_vma_walk->fault)

			goto fault;

			continue;

		return 0;

	}



	if (!pte_present(pte)) {
@@ -451,7 +422,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp, 
		if (!non_swap_entry(entry)) {

			if (hmm_vma_walk->fault)

				goto fault;

				continue;

			return 0;

		}



		/*
@@ -459,13 +430,16 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp, 
		 * device and report anything else as error.

		 */

		if (is_device_private_entry(entry)) {

				pfns[i] = hmm_pfn_from_pfn(swp_offset(entry));

			*pfn = hmm_pfn_from_pfn(swp_offset(entry));

			if (is_write_device_private_entry(entry)) {

					pfns[i] |= HMM_PFN_WRITE;

				} else if (write_fault)

				*pfn |= HMM_PFN_WRITE;

			} else if ((hmm_vma_walk->fault & hmm_vma_walk->write))

				goto fault;

				pfns[i] |= HMM_PFN_DEVICE_PRIVATE;

			} else if (is_migration_entry(entry)) {

			*pfn |= HMM_PFN_DEVICE_PRIVATE;

			return 0;

		}



		if (is_migration_entry(entry)) {

			if (hmm_vma_walk->fault) {

				pte_unmap(ptep);

				hmm_vma_walk->last = addr;
@@ -473,28 +447,84 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp, 
						pmdp, addr);

				return -EAGAIN;

			}

				continue;

			} else {

				/* Report error for everything else */

				pfns[i] = HMM_PFN_ERROR;

			return 0;

		}

			continue;



		/* Report error for everything else */

		*pfn = HMM_PFN_ERROR;

		return -EFAULT;

	}



		if (write_fault && !pte_write(pte))

	if ((hmm_vma_walk->fault & hmm_vma_walk->write) && !pte_write(pte))

		goto fault;



		pfns[i] = hmm_pfn_from_pfn(pte_pfn(pte));

		pfns[i] |= pte_write(pte) ? HMM_PFN_WRITE : 0;

		continue;

	*pfn = hmm_pfn_from_pfn(pte_pfn(pte));

	*pfn |= pte_write(pte) ? HMM_PFN_WRITE : 0;

	return 0;



fault:

	pte_unmap(ptep);

	/* Fault any virtual address we were asked to fault */

	return hmm_vma_walk_hole(addr, end, walk);

}



static int hmm_vma_walk_pmd(pmd_t *pmdp,

			    unsigned long start,

			    unsigned long end,

			    struct mm_walk *walk)

{

	struct hmm_vma_walk *hmm_vma_walk = walk->private;

	struct hmm_range *range = hmm_vma_walk->range;

	uint64_t *pfns = range->pfns;

	unsigned long addr = start, i;

	pte_t *ptep;



	i = (addr - range->start) >> PAGE_SHIFT;



again:

	if (pmd_none(*pmdp))

		return hmm_vma_walk_hole(start, end, walk);



	if (pmd_huge(*pmdp) && (range->vma->vm_flags & VM_HUGETLB))

		return hmm_pfns_bad(start, end, walk);



	if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {

		pmd_t pmd;



		/*

		 * No need to take pmd_lock here, even if some other threads

		 * is splitting the huge pmd we will get that event through

		 * mmu_notifier callback.

		 *

		 * So just read pmd value and check again its a transparent

		 * huge or device mapping one and compute corresponding pfn

		 * values.

		 */

		pmd = pmd_read_atomic(pmdp);

		barrier();

		if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))

			goto again;



		return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);

	}



	if (pmd_bad(*pmdp))

		return hmm_pfns_bad(start, end, walk);



	ptep = pte_offset_map(pmdp, addr);

	for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {

		int r;



		r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);

		if (r) {

			/* hmm_vma_handle_pte() did unmap pte directory */

			hmm_vma_walk->last = addr;

			return r;

		}

	}

	pte_unmap(ptep - 1);



	hmm_vma_walk->last = addr;

	return 0;

}