Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f8829cae authored by Ralf Baechle's avatar Ralf Baechle
Browse files

[MIPS] Fix aliasing bug in copy_to_user_page / copy_from_user_page 



The current implementation uses a sequence of a cacheflush and a copy.
This is racy in case of a multithreaded debuggee and renders GDB
virtually unusable.

Aside this fixes a performance hog rendering access to /proc/cmdline very
slow and resulting in a enough cache stalls for the 34K AP/SP programming
model to make the bare metal code on the non-Linux VPE miss RT deadlines.

The main part of this patch was originally written by Ralf Baechle;
Atushi Nemoto did the the debugging.

Signed-off-by: default avatarAtsushi Nemoto <anemo@mba.ocn.ne.jp>
Signed-off-by: default avatarRalf Baechle <ralf@linux-mips.org>
parent 224dc50e

arch/mips/mm/init.c

+160 −7
Original line number Diff line number Diff line
@@ -30,11 +30,34 @@ 
#include <asm/cachectl.h>

#include <asm/cpu.h>

#include <asm/dma.h>

#include <asm/kmap_types.h>

#include <asm/mmu_context.h>

#include <asm/sections.h>

#include <asm/pgtable.h>

#include <asm/pgalloc.h>

#include <asm/tlb.h>

#include <asm/fixmap.h>



/* Atomicity and interruptability */

#ifdef CONFIG_MIPS_MT_SMTC



#include <asm/mipsmtregs.h>



#define ENTER_CRITICAL(flags) \

	{ \

	unsigned int mvpflags; \

	local_irq_save(flags);\

	mvpflags = dvpe()

#define EXIT_CRITICAL(flags) \

	evpe(mvpflags); \

	local_irq_restore(flags); \

	}

#else



#define ENTER_CRITICAL(flags) local_irq_save(flags)

#define EXIT_CRITICAL(flags) local_irq_restore(flags)



#endif /* CONFIG_MIPS_MT_SMTC */



DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
@@ -80,13 +103,142 @@ unsigned long setup_zero_pages(void) 
	return 1UL << order;

}



#ifdef CONFIG_HIGHMEM

pte_t *kmap_pte;

pgprot_t kmap_prot;

/*

 * These are almost like kmap_atomic / kunmap_atmic except they take an

 * additional address argument as the hint.

 */



#define kmap_get_fixmap_pte(vaddr)					\

	pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr))



#ifdef CONFIG_MIPS_MT_SMTC

static pte_t *kmap_coherent_pte;

static void __init kmap_coherent_init(void)

{

	unsigned long vaddr;



	/* cache the first coherent kmap pte */

	vaddr = __fix_to_virt(FIX_CMAP_BEGIN);

	kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);

}

#else

static inline void kmap_coherent_init(void) {}

#endif



static inline void *kmap_coherent(struct page *page, unsigned long addr)

{

	enum fixed_addresses idx;

	unsigned long vaddr, flags, entrylo;

	unsigned long old_ctx;

	pte_t pte;

	int tlbidx;



	inc_preempt_count();

	idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);

#ifdef CONFIG_MIPS_MT_SMTC

	idx += FIX_N_COLOURS * smp_processor_id();

#endif

	vaddr = __fix_to_virt(FIX_CMAP_END - idx);

	pte = mk_pte(page, PAGE_KERNEL);

#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32_R1)

	entrylo = pte.pte_high;

#else

	entrylo = pte_val(pte) >> 6;

#endif



	ENTER_CRITICAL(flags);

	old_ctx = read_c0_entryhi();

	write_c0_entryhi(vaddr & (PAGE_MASK << 1));

	write_c0_entrylo0(entrylo);

	write_c0_entrylo1(entrylo);

#ifdef CONFIG_MIPS_MT_SMTC

	set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);

	/* preload TLB instead of local_flush_tlb_one() */

	mtc0_tlbw_hazard();

	tlb_probe();

	tlb_probe_hazard();

	tlbidx = read_c0_index();

	mtc0_tlbw_hazard();

	if (tlbidx < 0)

		tlb_write_random();

	else

		tlb_write_indexed();

#else

	tlbidx = read_c0_wired();

	write_c0_wired(tlbidx + 1);

	write_c0_index(tlbidx);

	mtc0_tlbw_hazard();

	tlb_write_indexed();

#endif

	tlbw_use_hazard();

	write_c0_entryhi(old_ctx);

	EXIT_CRITICAL(flags);



	return (void*) vaddr;

}



#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))



static inline void kunmap_coherent(struct page *page)

{

#ifndef CONFIG_MIPS_MT_SMTC

	unsigned int wired;

	unsigned long flags, old_ctx;



	ENTER_CRITICAL(flags);

	old_ctx = read_c0_entryhi();

	wired = read_c0_wired() - 1;

	write_c0_wired(wired);

	write_c0_index(wired);

	write_c0_entryhi(UNIQUE_ENTRYHI(wired));

	write_c0_entrylo0(0);

	write_c0_entrylo1(0);

	mtc0_tlbw_hazard();

	tlb_write_indexed();

	tlbw_use_hazard();

	write_c0_entryhi(old_ctx);

	EXIT_CRITICAL(flags);

#endif

	dec_preempt_count();

	preempt_check_resched();

}



void copy_to_user_page(struct vm_area_struct *vma,

	struct page *page, unsigned long vaddr, void *dst, const void *src,

	unsigned long len)

{

	if (cpu_has_dc_aliases) {

		void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);

		memcpy(vto, src, len);

		kunmap_coherent(page);

	} else

		memcpy(dst, src, len);

	if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)

		flush_cache_page(vma, vaddr, page_to_pfn(page));

}



EXPORT_SYMBOL(copy_to_user_page);



void copy_from_user_page(struct vm_area_struct *vma,

	struct page *page, unsigned long vaddr, void *dst, const void *src,

	unsigned long len)

{

	if (cpu_has_dc_aliases) {

		void *vfrom =

			kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);

		memcpy(dst, vfrom, len);

		kunmap_coherent(page);

	} else

		memcpy(dst, src, len);

}



EXPORT_SYMBOL(copy_from_user_page);





#ifdef CONFIG_HIGHMEM

pte_t *kmap_pte;

pgprot_t kmap_prot;



static void __init kmap_init(void)

{

	unsigned long kmap_vstart;
@@ -97,11 +249,12 @@ static void __init kmap_init(void) 


	kmap_prot = PAGE_KERNEL;

}

#endif /* CONFIG_HIGHMEM */



#ifdef CONFIG_32BIT

void __init fixrange_init(unsigned long start, unsigned long end,

	pgd_t *pgd_base)

{

#if defined(CONFIG_HIGHMEM) || defined(CONFIG_MIPS_MT_SMTC)

	pgd_t *pgd;

	pud_t *pud;

	pmd_t *pmd;
@@ -122,7 +275,7 @@ void __init fixrange_init(unsigned long start, unsigned long end, 
			for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {

				if (pmd_none(*pmd)) {

					pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);

					set_pmd(pmd, __pmd(pte));

					set_pmd(pmd, __pmd((unsigned long)pte));

					if (pte != pte_offset_kernel(pmd, 0))

						BUG();

				}
@@ -132,9 +285,8 @@ void __init fixrange_init(unsigned long start, unsigned long end, 
		}

		j = 0;

	}

#endif

}

#endif /* CONFIG_32BIT */

#endif /* CONFIG_HIGHMEM */



#ifndef CONFIG_NEED_MULTIPLE_NODES

extern void pagetable_init(void);
@@ -175,6 +327,7 @@ void __init paging_init(void) 
#ifdef CONFIG_HIGHMEM

	kmap_init();

#endif

	kmap_coherent_init();



	max_dma = virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT;

	low = max_low_pfn;

arch/mips/mm/pgtable-32.c

+4 −3
Original line number Diff line number Diff line
@@ -31,9 +31,10 @@ void pgd_init(unsigned long page) 


void __init pagetable_init(void)

{

#ifdef CONFIG_HIGHMEM

	unsigned long vaddr;

	pgd_t *pgd, *pgd_base;

	pgd_t *pgd_base;

#ifdef CONFIG_HIGHMEM

	pgd_t *pgd;

	pud_t *pud;

	pmd_t *pmd;

	pte_t *pte;
@@ -44,7 +45,6 @@ void __init pagetable_init(void) 
	pgd_init((unsigned long)swapper_pg_dir

		 + sizeof(pgd_t) * USER_PTRS_PER_PGD);



#ifdef CONFIG_HIGHMEM

	pgd_base = swapper_pg_dir;



	/*
@@ -53,6 +53,7 @@ void __init pagetable_init(void) 
	vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;

	fixrange_init(vaddr, 0, pgd_base);



#ifdef CONFIG_HIGHMEM

	/*

	 * Permanent kmaps:

	 */

arch/mips/mm/pgtable-64.c

+11 −0
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ 
 */

#include <linux/init.h>

#include <linux/mm.h>

#include <asm/fixmap.h>

#include <asm/pgtable.h>



void pgd_init(unsigned long page)
@@ -52,7 +53,17 @@ void pmd_init(unsigned long addr, unsigned long pagetable) 


void __init pagetable_init(void)

{

	unsigned long vaddr;

	pgd_t *pgd_base;



	/* Initialize the entire pgd.  */

	pgd_init((unsigned long)swapper_pg_dir);

	pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);



	pgd_base = swapper_pg_dir;

	/*

	 * Fixed mappings:

	 */

	vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;

	fixrange_init(vaddr, 0, pgd_base);

}

include/asm-mips/cacheflush.h

+4 −15
Original line number Diff line number Diff line
@@ -55,24 +55,13 @@ extern void (*flush_icache_range)(unsigned long start, unsigned long end); 
#define flush_cache_vmap(start, end)		flush_cache_all()

#define flush_cache_vunmap(start, end)		flush_cache_all()



static inline void copy_to_user_page(struct vm_area_struct *vma,

extern void copy_to_user_page(struct vm_area_struct *vma,

	struct page *page, unsigned long vaddr, void *dst, const void *src,

	unsigned long len)

{

	if (cpu_has_dc_aliases)

		flush_cache_page(vma, vaddr, page_to_pfn(page));

	memcpy(dst, src, len);

	__flush_icache_page(vma, page);

}

	unsigned long len);



static inline void copy_from_user_page(struct vm_area_struct *vma,

extern void copy_from_user_page(struct vm_area_struct *vma,

	struct page *page, unsigned long vaddr, void *dst, const void *src,

	unsigned long len)

{

	if (cpu_has_dc_aliases)

		flush_cache_page(vma, vaddr, page_to_pfn(page));

	memcpy(dst, src, len);

}

	unsigned long len);



extern void (*flush_cache_sigtramp)(unsigned long addr);

extern void (*flush_icache_all)(void);

include/asm-mips/fixmap.h

+11 −3
Original line number Diff line number Diff line
@@ -45,8 +45,16 @@ 
 * fix-mapped?

 */

enum fixed_addresses {

#define FIX_N_COLOURS 8

	FIX_CMAP_BEGIN,

#ifdef CONFIG_MIPS_MT_SMTC

	FIX_CMAP_END = FIX_CMAP_BEGIN + (FIX_N_COLOURS * NR_CPUS),

#else

	FIX_CMAP_END = FIX_CMAP_BEGIN + FIX_N_COLOURS,

#endif

#ifdef CONFIG_HIGHMEM

	FIX_KMAP_BEGIN,	/* reserved pte's for temporary kernel mappings */

	/* reserved pte's for temporary kernel mappings */

	FIX_KMAP_BEGIN = FIX_CMAP_END + 1,

	FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,

#endif

	__end_of_fixed_addresses
@@ -70,9 +78,9 @@ extern void __set_fixmap (enum fixed_addresses idx, 
 * at the top of mem..

 */

#if defined(CONFIG_CPU_TX39XX) || defined(CONFIG_CPU_TX49XX)

#define FIXADDR_TOP	(0xff000000UL - 0x2000)

#define FIXADDR_TOP	((unsigned long)(long)(int)(0xff000000 - 0x20000))

#else

#define FIXADDR_TOP	(0xffffe000UL)

#define FIXADDR_TOP	((unsigned long)(long)(int)0xfffe0000)

#endif

#define FIXADDR_SIZE	(__end_of_fixed_addresses << PAGE_SHIFT)

#define FIXADDR_START	(FIXADDR_TOP - FIXADDR_SIZE)