Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 34ee645e authored by Joerg Roedel's avatar Joerg Roedel Committed by Oded Gabbay
Browse files

mmu_notifier: call mmu_notifier_invalidate_range() from VMM 



Add calls to the new mmu_notifier_invalidate_range() function to all
places in the VMM that need it.

Signed-off-by: default avatarJoerg Roedel <jroedel@suse.de>
Reviewed-by: default avatarAndrea Arcangeli <aarcange@redhat.com>
Reviewed-by: default avatarJérôme Glisse <jglisse@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Rik van Riel <riel@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Johannes Weiner <jweiner@redhat.com>
Cc: Jay Cornwall <Jay.Cornwall@amd.com>
Cc: Oded Gabbay <Oded.Gabbay@amd.com>
Cc: Suravee Suthikulpanit <Suravee.Suthikulpanit@amd.com>
Cc: Jesse Barnes <jbarnes@virtuousgeek.org>
Cc: David Woodhouse <dwmw2@infradead.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarOded Gabbay <oded.gabbay@amd.com>
parent 1897bdc4

include/linux/mmu_notifier.h

+41 −0
Original line number Diff line number Diff line
@@ -284,6 +284,44 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) 
	__young;							\

})



#define	ptep_clear_flush_notify(__vma, __address, __ptep)		\

({									\

	unsigned long ___addr = __address & PAGE_MASK;			\

	struct mm_struct *___mm = (__vma)->vm_mm;			\

	pte_t ___pte;							\

									\

	___pte = ptep_clear_flush(__vma, __address, __ptep);		\

	mmu_notifier_invalidate_range(___mm, ___addr,			\

					___addr + PAGE_SIZE);		\

									\

	___pte;								\

})



#define pmdp_clear_flush_notify(__vma, __haddr, __pmd)			\

({									\

	unsigned long ___haddr = __haddr & HPAGE_PMD_MASK;		\

	struct mm_struct *___mm = (__vma)->vm_mm;			\

	pmd_t ___pmd;							\

									\

	___pmd = pmdp_clear_flush(__vma, __haddr, __pmd);		\

	mmu_notifier_invalidate_range(___mm, ___haddr,			\

				      ___haddr + HPAGE_PMD_SIZE);	\

									\

	___pmd;								\

})



#define pmdp_get_and_clear_notify(__mm, __haddr, __pmd)			\

({									\

	unsigned long ___haddr = __haddr & HPAGE_PMD_MASK;		\

	pmd_t ___pmd;							\

									\

	___pmd = pmdp_get_and_clear(__mm, __haddr, __pmd);		\

	mmu_notifier_invalidate_range(__mm, ___haddr,			\

				      ___haddr + HPAGE_PMD_SIZE);	\

									\

	___pmd;								\

})



/*

 * set_pte_at_notify() sets the pte _after_ running the notifier.

 * This is safe to start by updating the secondary MMUs, because the primary MMU
@@ -362,6 +400,9 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) 


#define ptep_clear_flush_young_notify ptep_clear_flush_young

#define pmdp_clear_flush_young_notify pmdp_clear_flush_young

#define	ptep_clear_flush_notify ptep_clear_flush

#define pmdp_clear_flush_notify pmdp_clear_flush

#define pmdp_get_and_clear_notify pmdp_get_and_clear

#define set_pte_at_notify set_pte_at



#endif /* CONFIG_MMU_NOTIFIER */

kernel/events/uprobes.c

+1 −1
Original line number Diff line number Diff line
@@ -193,7 +193,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, 
	}



	flush_cache_page(vma, addr, pte_pfn(*ptep));

	ptep_clear_flush(vma, addr, ptep);

	ptep_clear_flush_notify(vma, addr, ptep);

	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));



	page_remove_rmap(page);

mm/fremap.c

+1 −1
Original line number Diff line number Diff line
@@ -37,7 +37,7 @@ static void zap_pte(struct mm_struct *mm, struct vm_area_struct *vma, 


	if (pte_present(pte)) {

		flush_cache_page(vma, addr, pte_pfn(pte));

		pte = ptep_clear_flush(vma, addr, ptep);

		pte = ptep_clear_flush_notify(vma, addr, ptep);

		page = vm_normal_page(vma, addr, pte);

		if (page) {

			if (pte_dirty(pte))

mm/huge_memory.c

+5 −4
Original line number Diff line number Diff line
@@ -1036,7 +1036,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, 
		goto out_free_pages;

	VM_BUG_ON_PAGE(!PageHead(page), page);



	pmdp_clear_flush(vma, haddr, pmd);

	pmdp_clear_flush_notify(vma, haddr, pmd);

	/* leave pmd empty until pte is filled */



	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
@@ -1179,7 +1179,7 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 
		pmd_t entry;

		entry = mk_huge_pmd(new_page, vma->vm_page_prot);

		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);

		pmdp_clear_flush(vma, haddr, pmd);

		pmdp_clear_flush_notify(vma, haddr, pmd);

		page_add_new_anon_rmap(new_page, vma, haddr);

		mem_cgroup_commit_charge(new_page, memcg, false);

		lru_cache_add_active_or_unevictable(new_page, vma);
@@ -1512,7 +1512,7 @@ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 
		pmd_t entry;

		ret = 1;

		if (!prot_numa) {

			entry = pmdp_get_and_clear(mm, addr, pmd);

			entry = pmdp_get_and_clear_notify(mm, addr, pmd);

			if (pmd_numa(entry))

				entry = pmd_mknonnuma(entry);

			entry = pmd_modify(entry, newprot);
@@ -1644,6 +1644,7 @@ static int __split_huge_page_splitting(struct page *page, 
		 * serialize against split_huge_page*.

		 */

		pmdp_splitting_flush(vma, address, pmd);



		ret = 1;

		spin_unlock(ptl);

	}
@@ -2834,7 +2835,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 
	pmd_t _pmd;

	int i;



	pmdp_clear_flush(vma, haddr, pmd);

	pmdp_clear_flush_notify(vma, haddr, pmd);

	/* leave pmd empty until pte is filled */



	pgtable = pgtable_trans_huge_withdraw(mm, pmd);

mm/hugetlb.c

+6 −1
Original line number Diff line number Diff line
@@ -2598,8 +2598,11 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, 
			}

			set_huge_pte_at(dst, addr, dst_pte, entry);

		} else {

			if (cow)

			if (cow) {

				huge_ptep_set_wrprotect(src, addr, src_pte);

				mmu_notifier_invalidate_range(src, mmun_start,

								   mmun_end);

			}

			entry = huge_ptep_get(src_pte);

			ptepage = pte_page(entry);

			get_page(ptepage);
@@ -2899,6 +2902,7 @@ static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, 


		/* Break COW */

		huge_ptep_clear_flush(vma, address, ptep);

		mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);

		set_huge_pte_at(mm, address, ptep,

				make_huge_pte(vma, new_page, 1));

		page_remove_rmap(old_page);
@@ -3374,6 +3378,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, 
	 * and that page table be reused and filled with junk.

	 */

	flush_tlb_range(vma, start, end);

	mmu_notifier_invalidate_range(mm, start, end);

	mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);

	mmu_notifier_invalidate_range_end(mm, start, end);