Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8cb8140c authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Michael Ellerman
Browse files

powerpc/mm/radix: Implement tlb mmu gather flush efficiently 



Now that we track page size in mmu_gather, we can use address based
tlbie format when doing a tlb_flush(). We don't do this if we are
invalidating the full address space.

Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent e2985fd9

arch/powerpc/include/asm/book3s/64/tlbflush-radix.h

+2 −0
Original line number Diff line number Diff line
@@ -10,6 +10,8 @@ static inline int mmu_get_ap(int psize) 
	return mmu_psize_defs[psize].ap;

}



extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,

					 unsigned long end, int psize);

extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,

			    unsigned long end);

extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);

arch/powerpc/mm/tlb-radix.c

+58 −1
Original line number Diff line number Diff line
@@ -299,10 +299,67 @@ static int radix_get_mmu_psize(int page_size) 


void radix__tlb_flush(struct mmu_gather *tlb)

{

	int psize = 0;

	struct mm_struct *mm = tlb->mm;

	int page_size = tlb->page_size;



	psize = radix_get_mmu_psize(page_size);

	/*

	 * if page size is not something we understand, do a full mm flush

	 */

	if (psize != -1 && !tlb->fullmm && !tlb->need_flush_all)

		radix__flush_tlb_range_psize(mm, tlb->start, tlb->end, psize);

	else

		radix__flush_tlb_mm(mm);

}



#define TLB_FLUSH_ALL -1UL

/*

 * Number of pages above which we will do a bcast tlbie. Just a

 * number at this point copied from x86

 */

static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;



void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,

				  unsigned long end, int psize)

{

	unsigned long pid;

	unsigned long addr;

	int local = mm_is_core_local(mm);

	unsigned long ap = mmu_get_ap(psize);

	int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);

	unsigned long page_size = 1UL << mmu_psize_defs[psize].shift;





	preempt_disable();

	pid = mm ? mm->context.id : 0;

	if (unlikely(pid == MMU_NO_CONTEXT))

		goto err_out;



	if (end == TLB_FLUSH_ALL ||

	    (end - start) > tlb_single_page_flush_ceiling * page_size) {

		if (local)

			_tlbiel_pid(pid, RIC_FLUSH_TLB);

		else

			_tlbie_pid(pid, RIC_FLUSH_TLB);

		goto err_out;

	}

	for (addr = start; addr < end; addr += page_size) {



		if (local)

			_tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);

		else {

			if (lock_tlbie)

				raw_spin_lock(&native_tlbie_lock);

			_tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);

			if (lock_tlbie)

				raw_spin_unlock(&native_tlbie_lock);

		}

	}

err_out:

	preempt_enable();

}



void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,

			      unsigned long page_size)

{