Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f048aace authored by Benjamin Herrenschmidt's avatar Benjamin Herrenschmidt Committed by Paul Mackerras
Browse files

powerpc/mm: Add SMP support to no-hash TLB handling 



This commit moves the whole no-hash TLB handling out of line into a
new tlb_nohash.c file, and implements some basic SMP support using
IPIs and/or broadcast tlbivax instructions.

Note that I'm using local invalidations for D->I cache coherency.

At worst, if another processor is trying to execute the same and
has the old entry in its TLB, it will just take a fault and re-do
the TLB flush locally (it won't re-do the cache flush in any case).

Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: default avatarKumar Gala <galak@kernel.crashing.org>
Signed-off-by: default avatarPaul Mackerras <paulus@samba.org>
parent 7c03d653

arch/powerpc/include/asm/highmem.h

+2 −2
Original line number Diff line number Diff line
@@ -85,7 +85,7 @@ static inline void *kmap_atomic_prot(struct page *page, enum km_type type, pgpro 
	BUG_ON(!pte_none(*(kmap_pte-idx)));

#endif

	__set_pte_at(&init_mm, vaddr, kmap_pte-idx, mk_pte(page, prot));

	local_flush_tlb_page(vaddr);

	local_flush_tlb_page(NULL, vaddr);



	return (void*) vaddr;

}
@@ -113,7 +113,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type) 
	 * this pte without first remap it

	 */

	pte_clear(&init_mm, vaddr, kmap_pte-idx);

	local_flush_tlb_page(vaddr);

	local_flush_tlb_page(NULL, vaddr);

#endif

	pagefault_enable();

}

arch/powerpc/include/asm/mmu.h

+16 −0
Original line number Diff line number Diff line
@@ -30,6 +30,22 @@ 
 */

#define MMU_FTR_BIG_PHYS		ASM_CONST(0x00020000)



/* Enable use of broadcast TLB invalidations. We don't always set it

 * on processors that support it due to other constraints with the

 * use of such invalidations

 */

#define MMU_FTR_USE_TLBIVAX_BCAST	ASM_CONST(0x00040000)



/* Enable use of tlbilx invalidate-by-PID variant.

 */

#define MMU_FTR_USE_TLBILX_PID		ASM_CONST(0x00080000)



/* This indicates that the processor cannot handle multiple outstanding

 * broadcast tlbivax or tlbsync. This makes the code use a spinlock

 * around such invalidate forms.

 */

#define MMU_FTR_LOCK_BCAST_INVAL	ASM_CONST(0x00100000)



#ifndef __ASSEMBLY__

#include <asm/cputable.h>

arch/powerpc/include/asm/tlbflush.h

+38 −46
Original line number Diff line number Diff line
@@ -6,7 +6,9 @@ 
 *

 *  - flush_tlb_mm(mm) flushes the specified mm context TLB's

 *  - flush_tlb_page(vma, vmaddr) flushes one page

 *  - local_flush_tlb_page(vmaddr) flushes one page on the local processor

 *  - local_flush_tlb_mm(mm) flushes the specified mm context on

 *                           the local processor

 *  - local_flush_tlb_page(vma, vmaddr) flushes one page on the local processor

 *  - flush_tlb_page_nohash(vma, vmaddr) flushes one page if SW loaded TLB

 *  - flush_tlb_range(vma, start, end) flushes a range of pages

 *  - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
@@ -18,7 +20,7 @@ 
 */

#ifdef __KERNEL__



#if defined(CONFIG_4xx) || defined(CONFIG_8xx) || defined(CONFIG_FSL_BOOKE)

#ifdef CONFIG_PPC_MMU_NOHASH

/*

 * TLB flushing for software loaded TLB chips

 *
@@ -31,10 +33,10 @@ 


#define MMU_NO_CONTEXT      	((unsigned int)-1)



extern void _tlbie(unsigned long address, unsigned int pid);

extern void _tlbil_all(void);

extern void _tlbil_pid(unsigned int pid);

extern void _tlbil_va(unsigned long address, unsigned int pid);

extern void _tlbivax_bcast(unsigned long address, unsigned int pid);



#if defined(CONFIG_40x) || defined(CONFIG_8xx)

#define _tlbia()	asm volatile ("tlbia; sync" : : : "memory")
@@ -42,48 +44,26 @@ extern void _tlbil_va(unsigned long address, unsigned int pid); 
extern void _tlbia(void);

#endif



static inline void local_flush_tlb_mm(struct mm_struct *mm)

{

	_tlbil_pid(mm->context.id);

}



static inline void flush_tlb_mm(struct mm_struct *mm)

{

	_tlbil_pid(mm->context.id);

}



static inline void local_flush_tlb_page(unsigned long vmaddr)

{

	_tlbil_va(vmaddr, 0);

}



static inline void flush_tlb_page(struct vm_area_struct *vma,

				  unsigned long vmaddr)

{

	_tlbil_va(vmaddr, vma ? vma->vm_mm->context.id : 0);

}

extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,

			    unsigned long end);

extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);



static inline void flush_tlb_page_nohash(struct vm_area_struct *vma,

					 unsigned long vmaddr)

{

	flush_tlb_page(vma, vmaddr);

}

extern void local_flush_tlb_mm(struct mm_struct *mm);

extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);



static inline void flush_tlb_range(struct vm_area_struct *vma,

				   unsigned long start, unsigned long end)

{

	_tlbil_pid(vma->vm_mm->context.id);

}

#ifdef CONFIG_SMP

extern void flush_tlb_mm(struct mm_struct *mm);

extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);

#else

#define flush_tlb_mm(mm)		local_flush_tlb_mm(mm)

#define flush_tlb_page(vma,addr)	local_flush_tlb_page(vma,addr)

#endif

#define flush_tlb_page_nohash(vma,addr)	flush_tlb_page(vma,addr)



static inline void flush_tlb_kernel_range(unsigned long start,

					  unsigned long end)

{

	_tlbil_pid(0);

}

#elif defined(CONFIG_PPC_STD_MMU_32)



#elif defined(CONFIG_PPC32)

/*

 * TLB flushing for "classic" hash-MMMU 32-bit CPUs, 6xx, 7xx, 7xxx

 * TLB flushing for "classic" hash-MMU 32-bit CPUs, 6xx, 7xx, 7xxx

 */

extern void _tlbie(unsigned long address);

extern void _tlbia(void);
@@ -94,14 +74,20 @@ extern void flush_tlb_page_nohash(struct vm_area_struct *vma, unsigned long addr 
extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,

			    unsigned long end);

extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);

static inline void local_flush_tlb_page(unsigned long vmaddr)

static inline void local_flush_tlb_page(struct vm_area_struct *vma,

					unsigned long vmaddr)

{

	flush_tlb_page(vma, vmaddr);

}

static inline void local_flush_tlb_mm(struct mm_struct *mm)

{

	flush_tlb_page(NULL, vmaddr);

	flush_tlb_mm(mm);

}



#else

#elif defined(CONFIG_PPC_STD_MMU_64)



/*

 * TLB flushing for 64-bit has-MMU CPUs

 * TLB flushing for 64-bit hash-MMU CPUs

 */



#include <linux/percpu.h>
@@ -151,11 +137,16 @@ extern void flush_hash_page(unsigned long va, real_pte_t pte, int psize, 
extern void flush_hash_range(unsigned long number, int local);





static inline void local_flush_tlb_mm(struct mm_struct *mm)

{

}



static inline void flush_tlb_mm(struct mm_struct *mm)

{

}



static inline void local_flush_tlb_page(unsigned long vmaddr)

static inline void local_flush_tlb_page(struct vm_area_struct *vma,

					unsigned long vmaddr)

{

}
@@ -183,7 +174,8 @@ static inline void flush_tlb_kernel_range(unsigned long start, 
extern void __flush_hash_table_range(struct mm_struct *mm, unsigned long start,

				     unsigned long end);





#else

#error Unsupported MMU type

#endif



#endif /*__KERNEL__ */

arch/powerpc/kernel/misc_32.S

+9 −0
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ 
#include <asm/asm-offsets.h>

#include <asm/processor.h>

#include <asm/kexec.h>

#include <asm/bug.h>



	.text
@@ -496,6 +497,14 @@ _GLOBAL(_tlbil_va) 
	blr

#endif /* CONFIG_FSL_BOOKE */



/*

 * Nobody implements this yet

 */

_GLOBAL(_tlbivax_bcast)

1:	trap

	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0;

	blr





/*

 * Flush instruction cache.

arch/powerpc/kernel/ppc_ksyms.c

+0 −6
Original line number Diff line number Diff line
@@ -116,12 +116,6 @@ EXPORT_SYMBOL(giveup_spe); 


#ifndef CONFIG_PPC64

EXPORT_SYMBOL(flush_instruction_cache);

EXPORT_SYMBOL(flush_tlb_kernel_range);

EXPORT_SYMBOL(flush_tlb_page);

EXPORT_SYMBOL(_tlbie);

#if defined(CONFIG_4xx) || defined(CONFIG_8xx) || defined(CONFIG_FSL_BOOKE)

EXPORT_SYMBOL(_tlbil_va);

#endif

#endif

EXPORT_SYMBOL(__flush_icache_range);

EXPORT_SYMBOL(flush_dcache_range);