Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e7891b22 authored by Peter Zijlstra (Intel)'s avatar Peter Zijlstra (Intel) Committed by Greg Kroah-Hartman
Browse files

tlb: mmu_gather: add tlb_flush_*_range APIs 



commit 2631ed00b0498810f8d5c2163c6b5270d893687b upstream.

tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and
tlb->end, then set corresponding cleared_*.

Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: default avatarZhenyu Ye <yezhenyu2@huawei.com>
Acked-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20200625080314.230-5-yezhenyu2@huawei.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent 10e34766

include/asm-generic/tlb.h

+40 −15
Original line number Diff line number Diff line
@@ -495,6 +495,38 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 
}

#endif



/*

 * tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end,

 * and set corresponding cleared_*.

 */

static inline void tlb_flush_pte_range(struct mmu_gather *tlb,

				     unsigned long address, unsigned long size)

{

	__tlb_adjust_range(tlb, address, size);

	tlb->cleared_ptes = 1;

}



static inline void tlb_flush_pmd_range(struct mmu_gather *tlb,

				     unsigned long address, unsigned long size)

{

	__tlb_adjust_range(tlb, address, size);

	tlb->cleared_pmds = 1;

}



static inline void tlb_flush_pud_range(struct mmu_gather *tlb,

				     unsigned long address, unsigned long size)

{

	__tlb_adjust_range(tlb, address, size);

	tlb->cleared_puds = 1;

}



static inline void tlb_flush_p4d_range(struct mmu_gather *tlb,

				     unsigned long address, unsigned long size)

{

	__tlb_adjust_range(tlb, address, size);

	tlb->cleared_p4ds = 1;

}



#ifndef __tlb_remove_tlb_entry

#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)

#endif
@@ -508,19 +540,17 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 
 */

#define tlb_remove_tlb_entry(tlb, ptep, address)		\

	do {							\

		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\

		tlb->cleared_ptes = 1;				\

		tlb_flush_pte_range(tlb, address, PAGE_SIZE);	\

		__tlb_remove_tlb_entry(tlb, ptep, address);	\

	} while (0)



#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\

	do {							\

		unsigned long _sz = huge_page_size(h);		\

		__tlb_adjust_range(tlb, address, _sz);		\

		if (_sz == PMD_SIZE)				\

			tlb->cleared_pmds = 1;			\

			tlb_flush_pmd_range(tlb, address, _sz);	\

		else if (_sz == PUD_SIZE)			\

			tlb->cleared_puds = 1;			\

			tlb_flush_pud_range(tlb, address, _sz);	\

		__tlb_remove_tlb_entry(tlb, ptep, address);	\

	} while (0)
@@ -534,8 +564,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 


#define tlb_remove_pmd_tlb_entry(tlb, pmdp, address)			\

	do {								\

		__tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE);	\

		tlb->cleared_pmds = 1;					\

		tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE);	\

		__tlb_remove_pmd_tlb_entry(tlb, pmdp, address);		\

	} while (0)
@@ -549,8 +578,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 


#define tlb_remove_pud_tlb_entry(tlb, pudp, address)			\

	do {								\

		__tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE);	\

		tlb->cleared_puds = 1;					\

		tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE);	\

		__tlb_remove_pud_tlb_entry(tlb, pudp, address);		\

	} while (0)
@@ -575,9 +603,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 
#ifndef pte_free_tlb

#define pte_free_tlb(tlb, ptep, address)			\

	do {							\

		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\

		tlb_flush_pmd_range(tlb, address, PAGE_SIZE);	\

		tlb->freed_tables = 1;				\

		tlb->cleared_pmds = 1;				\

		__pte_free_tlb(tlb, ptep, address);		\

	} while (0)

#endif
@@ -585,9 +612,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 
#ifndef pmd_free_tlb

#define pmd_free_tlb(tlb, pmdp, address)			\

	do {							\

		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\

		tlb_flush_pud_range(tlb, address, PAGE_SIZE);	\

		tlb->freed_tables = 1;				\

		tlb->cleared_puds = 1;				\

		__pmd_free_tlb(tlb, pmdp, address);		\

	} while (0)

#endif
@@ -596,9 +622,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm 
#ifndef pud_free_tlb

#define pud_free_tlb(tlb, pudp, address)			\

	do {							\

		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\

		tlb_flush_p4d_range(tlb, address, PAGE_SIZE);	\

		tlb->freed_tables = 1;				\

		tlb->cleared_p4ds = 1;				\

		__pud_free_tlb(tlb, pudp, address);		\

	} while (0)

#endif