Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d281ee61 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds
Browse files

rmap: add argument to charge compound page 



We're going to allow mapping of individual 4k pages of THP compound
page.  It means we cannot rely on PageTransHuge() check to decide if
map/unmap small page or THP.

The patch adds new argument to rmap functions to indicate whether we
want to operate on whole compound page or only the small page.

[n-horiguchi@ah.jp.nec.com: fix mapcount mismatch in hugepage migration]
Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Tested-by: default avatarSasha Levin <sasha.levin@oracle.com>
Tested-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarJerome Marchand <jmarchan@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Steve Capper <steve.capper@linaro.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: default avatarNaoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent afd9883f

include/linux/rmap.h

+9 −3
Original line number Diff line number Diff line
@@ -161,16 +161,22 @@ static inline void anon_vma_merge(struct vm_area_struct *vma, 


struct anon_vma *page_get_anon_vma(struct page *page);



/* bitflags for do_page_add_anon_rmap() */

#define RMAP_EXCLUSIVE 0x01

#define RMAP_COMPOUND 0x02



/*

 * rmap interfaces called when adding or removing pte of page

 */

void page_move_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);

void page_add_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);

void page_add_anon_rmap(struct page *, struct vm_area_struct *,

		unsigned long, bool);

void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,

			   unsigned long, int);

void page_add_new_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);

void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,

		unsigned long, bool);

void page_add_file_rmap(struct page *);

void page_remove_rmap(struct page *);

void page_remove_rmap(struct page *, bool);



void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,

			    unsigned long);

kernel/events/uprobes.c

+2 −2
Original line number Diff line number Diff line
@@ -175,7 +175,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 
		goto unlock;



	get_page(kpage);

	page_add_new_anon_rmap(kpage, vma, addr);

	page_add_new_anon_rmap(kpage, vma, addr, false);

	mem_cgroup_commit_charge(kpage, memcg, false);

	lru_cache_add_active_or_unevictable(kpage, vma);
@@ -188,7 +188,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 
	ptep_clear_flush_notify(vma, addr, ptep);

	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));



	page_remove_rmap(page);

	page_remove_rmap(page, false);

	if (!page_mapped(page))

		try_to_free_swap(page);

	pte_unmap_unlock(ptep, ptl);

mm/huge_memory.c

+8 −8
Original line number Diff line number Diff line
@@ -797,7 +797,7 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, 


		entry = mk_huge_pmd(page, vma->vm_page_prot);

		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);

		page_add_new_anon_rmap(page, vma, haddr);

		page_add_new_anon_rmap(page, vma, haddr, true);

		mem_cgroup_commit_charge(page, memcg, false);

		lru_cache_add_active_or_unevictable(page, vma);

		pgtable_trans_huge_deposit(mm, pmd, pgtable);
@@ -1139,7 +1139,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, 
		entry = maybe_mkwrite(pte_mkdirty(entry), vma);

		memcg = (void *)page_private(pages[i]);

		set_page_private(pages[i], 0);

		page_add_new_anon_rmap(pages[i], vma, haddr);

		page_add_new_anon_rmap(pages[i], vma, haddr, false);

		mem_cgroup_commit_charge(pages[i], memcg, false);

		lru_cache_add_active_or_unevictable(pages[i], vma);

		pte = pte_offset_map(&_pmd, haddr);
@@ -1151,7 +1151,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, 


	smp_wmb(); /* make pte visible before pmd */

	pmd_populate(mm, pmd, pgtable);

	page_remove_rmap(page);

	page_remove_rmap(page, true);

	spin_unlock(ptl);



	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
@@ -1271,7 +1271,7 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 
		entry = mk_huge_pmd(new_page, vma->vm_page_prot);

		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);

		pmdp_huge_clear_flush_notify(vma, haddr, pmd);

		page_add_new_anon_rmap(new_page, vma, haddr);

		page_add_new_anon_rmap(new_page, vma, haddr, true);

		mem_cgroup_commit_charge(new_page, memcg, false);

		lru_cache_add_active_or_unevictable(new_page, vma);

		set_pmd_at(mm, haddr, pmd, entry);
@@ -1281,7 +1281,7 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 
			put_huge_zero_page();

		} else {

			VM_BUG_ON_PAGE(!PageHead(page), page);

			page_remove_rmap(page);

			page_remove_rmap(page, true);

			put_page(page);

		}

		ret |= VM_FAULT_WRITE;
@@ -1508,7 +1508,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 
		put_huge_zero_page();

	} else {

		struct page *page = pmd_page(orig_pmd);

		page_remove_rmap(page);

		page_remove_rmap(page, true);

		VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);

		add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);

		VM_BUG_ON_PAGE(!PageHead(page), page);
@@ -2371,7 +2371,7 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 
			 * superfluous.

			 */

			pte_clear(vma->vm_mm, address, _pte);

			page_remove_rmap(src_page);

			page_remove_rmap(src_page, false);

			spin_unlock(ptl);

			free_page_and_swap_cache(src_page);

		}
@@ -2682,7 +2682,7 @@ static void collapse_huge_page(struct mm_struct *mm, 


	spin_lock(pmd_ptl);

	BUG_ON(!pmd_none(*pmd));

	page_add_new_anon_rmap(new_page, vma, address);

	page_add_new_anon_rmap(new_page, vma, address, true);

	mem_cgroup_commit_charge(new_page, memcg, false);

	lru_cache_add_active_or_unevictable(new_page, vma);

	pgtable_trans_huge_deposit(mm, pmd, pgtable);

mm/hugetlb.c

+2 −2
Original line number Diff line number Diff line
@@ -3186,7 +3186,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, 
			set_page_dirty(page);



		hugetlb_count_sub(pages_per_huge_page(h), mm);

		page_remove_rmap(page);

		page_remove_rmap(page, true);

		force_flush = !__tlb_remove_page(tlb, page);

		if (force_flush) {

			address += sz;
@@ -3415,7 +3415,7 @@ static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 
		mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);

		set_huge_pte_at(mm, address, ptep,

				make_huge_pte(vma, new_page, 1));

		page_remove_rmap(old_page);

		page_remove_rmap(old_page, true);

		hugepage_add_new_anon_rmap(new_page, vma, address);

		/* Make the old page be freed below */

		new_page = old_page;

mm/ksm.c

+2 −2
Original line number Diff line number Diff line
@@ -956,13 +956,13 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, 
	}



	get_page(kpage);

	page_add_anon_rmap(kpage, vma, addr);

	page_add_anon_rmap(kpage, vma, addr, false);



	flush_cache_page(vma, addr, pte_pfn(*ptep));

	ptep_clear_flush_notify(vma, addr, ptep);

	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));



	page_remove_rmap(page);

	page_remove_rmap(page, false);

	if (!page_mapped(page))

		try_to_free_swap(page);

	put_page(page);