Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 03319327 authored by Dave Hansen's avatar Dave Hansen Committed by Linus Torvalds
Browse files

pagewalk: only split huge pages when necessary 



Right now, if a mm_walk has either ->pte_entry or ->pmd_entry set, it will
unconditionally split any transparent huge pages it runs in to.  In
practice, that means that anyone doing a

	cat /proc/$pid/smaps

will unconditionally break down every huge page in the process and depend
on khugepaged to re-collapse it later.  This is fairly suboptimal.

This patch changes that behavior.  It teaches each ->pmd_entry handler
(there are five) that they must break down the THPs themselves.  Also, the
_generic_ code will never break down a THP unless a ->pte_entry handler is
actually set.

This means that the ->pmd_entry handlers can now choose to deal with THPs
without breaking them down.

[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: default avatarDave Hansen <dave@linux.vnet.ibm.com>
Acked-by: default avatarMel Gorman <mel@csn.ul.ie>
Acked-by: default avatarDavid Rientjes <rientjes@google.com>
Reviewed-by: default avatarEric B Munson <emunson@mgebm.net>
Tested-by: default avatarEric B Munson <emunson@mgebm.net>
Cc: Michael J Wolf <mjwolf@us.ibm.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Matt Mackall <mpm@selenic.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 278df9f4

fs/proc/task_mmu.c

+6 −0
Original line number Original line Diff line number Diff line
@@ -343,6 +343,8 @@ static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 
	struct page *page;

	struct page *page;

	int mapcount;

	int mapcount;





	split_huge_page_pmd(walk->mm, pmd);



	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);

	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);

	for (; addr != end; pte++, addr += PAGE_SIZE) {

	for (; addr != end; pte++, addr += PAGE_SIZE) {

		ptent = *pte;

		ptent = *pte;
@@ -467,6 +469,8 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, 
	spinlock_t *ptl;

	spinlock_t *ptl;

	struct page *page;

	struct page *page;





	split_huge_page_pmd(walk->mm, pmd);



	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);

	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);

	for (; addr != end; pte++, addr += PAGE_SIZE) {

	for (; addr != end; pte++, addr += PAGE_SIZE) {

		ptent = *pte;

		ptent = *pte;
@@ -623,6 +627,8 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 
	pte_t *pte;

	pte_t *pte;

	int err = 0;

	int err = 0;





	split_huge_page_pmd(walk->mm, pmd);



	/* find the first VMA at or above 'addr' */

	/* find the first VMA at or above 'addr' */

	vma = find_vma(walk->mm, addr);

	vma = find_vma(walk->mm, addr);

	for (; addr != end; addr += PAGE_SIZE) {

	for (; addr != end; addr += PAGE_SIZE) {

include/linux/mm.h

+3 −0
Original line number Original line Diff line number Diff line
@@ -914,6 +914,9 @@ unsigned long unmap_vmas(struct mmu_gather **tlb, 
 * @pgd_entry: if set, called for each non-empty PGD (top-level) entry

 * @pgd_entry: if set, called for each non-empty PGD (top-level) entry

 * @pud_entry: if set, called for each non-empty PUD (2nd-level) entry

 * @pud_entry: if set, called for each non-empty PUD (2nd-level) entry

 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry

 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry

 *	       this handler is required to be able to handle

 *	       pmd_trans_huge() pmds.  They may simply choose to

 *	       split_huge_page() instead of handling it explicitly.

 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry

 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry

 * @pte_hole: if set, called for each hole at all levels

 * @pte_hole: if set, called for each hole at all levels

 * @hugetlb_entry: if set, called for each hugetlb entry

 * @hugetlb_entry: if set, called for each hugetlb entry

mm/memcontrol.c

+3 −2
Original line number Original line Diff line number Diff line
@@ -4763,7 +4763,8 @@ static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, 
	pte_t *pte;

	pte_t *pte;

	spinlock_t *ptl;

	spinlock_t *ptl;





	VM_BUG_ON(pmd_trans_huge(*pmd));

	split_huge_page_pmd(walk->mm, pmd);



	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);

	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);

	for (; addr != end; pte++, addr += PAGE_SIZE)

	for (; addr != end; pte++, addr += PAGE_SIZE)

		if (is_target_pte_for_mc(vma, addr, *pte, NULL))

		if (is_target_pte_for_mc(vma, addr, *pte, NULL))
@@ -4925,8 +4926,8 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, 
	pte_t *pte;

	pte_t *pte;

	spinlock_t *ptl;

	spinlock_t *ptl;





	split_huge_page_pmd(walk->mm, pmd);

retry:

retry:

	VM_BUG_ON(pmd_trans_huge(*pmd));

	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);

	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);

	for (; addr != end; addr += PAGE_SIZE) {

	for (; addr != end; addr += PAGE_SIZE) {

		pte_t ptent = *(pte++);

		pte_t ptent = *(pte++);

mm/pagewalk.c

+20 −4
Original line number Original line Diff line number Diff line
@@ -33,18 +33,34 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 




	pmd = pmd_offset(pud, addr);

	pmd = pmd_offset(pud, addr);

	do {

	do {

again:

		next = pmd_addr_end(addr, end);

		next = pmd_addr_end(addr, end);

		split_huge_page_pmd(walk->mm, pmd);

		if (pmd_none(*pmd)) {

		if (pmd_none_or_clear_bad(pmd)) {

			if (walk->pte_hole)

			if (walk->pte_hole)

				err = walk->pte_hole(addr, next, walk);

				err = walk->pte_hole(addr, next, walk);

			if (err)

			if (err)

				break;

				break;

			continue;

			continue;

		}

		}

		/*

		 * This implies that each ->pmd_entry() handler

		 * needs to know about pmd_trans_huge() pmds

		 */

		if (walk->pmd_entry)

		if (walk->pmd_entry)

			err = walk->pmd_entry(pmd, addr, next, walk);

			err = walk->pmd_entry(pmd, addr, next, walk);

		if (!err && walk->pte_entry)

		if (err)

			break;



		/*

		 * Check this here so we only break down trans_huge

		 * pages when we _need_ to

		 */

		if (!walk->pte_entry)

			continue;



		split_huge_page_pmd(walk->mm, pmd);

		if (pmd_none_or_clear_bad(pmd))

			goto again;

		err = walk_pte_range(pmd, addr, next, walk);

		err = walk_pte_range(pmd, addr, next, walk);

		if (err)

		if (err)

			break;

			break;