Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6656920b authored by Chris Zankel's avatar Chris Zankel
Browse files

[XTENSA] Add support for cache-aliasing 



Add support for processors that have cache-aliasing issues, such as
the Stretch S5000 processor. Cache-aliasing means that the size of
the cache (for one way) is larger than the page size, thus, a page
can end up in several places in cache depending on the virtual to
physical translation. The method used here is to map a user page
temporarily through the auto-refill way 0 and of of the DTLB.
We probably will want to revisit this issue and use a better
approach with kmap/kunmap.

Signed-off-by: default avatarChris Zankel <chris@zankel.net>
parent ff6fd469

arch/xtensa/kernel/asm-offsets.c

+6 −7
Original line number Diff line number Diff line
@@ -18,12 +18,13 @@ 
#include <linux/stddef.h>

#include <linux/thread_info.h>

#include <linux/ptrace.h>

#include <linux/mm.h>



#include <asm/ptrace.h>

#include <asm/processor.h>

#include <asm/uaccess.h>



#define DEFINE(sym, val) asm volatile("\n->" #sym " %0 " #val : : "i" (val))

#define BLANK() asm volatile("\n->" : : )



int main(void)

{
@@ -63,7 +64,6 @@ int main(void) 
	DEFINE(PT_SIZE, sizeof(struct pt_regs));

	DEFINE(PT_AREG_END, offsetof (struct pt_regs, areg[XCHAL_NUM_AREGS]));

	DEFINE(PT_USER_SIZE, offsetof(struct pt_regs, areg[XCHAL_NUM_AREGS]));

	BLANK();



	/* struct task_struct */

	DEFINE(TASK_PTRACE, offsetof (struct task_struct, ptrace));
@@ -73,27 +73,26 @@ int main(void) 
	DEFINE(TASK_THREAD, offsetof (struct task_struct, thread));

	DEFINE(TASK_THREAD_INFO, offsetof (struct task_struct, stack));

	DEFINE(TASK_STRUCT_SIZE, sizeof (struct task_struct));

	BLANK();



	/* struct thread_info (offset from start_struct) */

	DEFINE(THREAD_RA, offsetof (struct task_struct, thread.ra));

	DEFINE(THREAD_SP, offsetof (struct task_struct, thread.sp));

	DEFINE(THREAD_CP_SAVE, offsetof (struct task_struct, thread.cp_save));

	DEFINE(THREAD_CURRENT_DS, offsetof (struct task_struct, thread.current_ds));

	BLANK();



	/* struct mm_struct */

	DEFINE(MM_USERS, offsetof(struct mm_struct, mm_users));

	DEFINE(MM_PGD, offsetof (struct mm_struct, pgd));

	DEFINE(MM_CONTEXT, offsetof (struct mm_struct, context));

	BLANK();

	DEFINE(PT_SINGLESTEP_BIT, PT_SINGLESTEP_BIT);



	/* struct page */

	DEFINE(PAGE_FLAGS, offsetof(struct page, flags));



	/* constants */

	DEFINE(_CLONE_VM, CLONE_VM);

	DEFINE(_CLONE_UNTRACED, CLONE_UNTRACED);

	DEFINE(PG_ARCH_1, PG_arch_1);



	return 0;

}

arch/xtensa/kernel/entry.S

+71 −4
Original line number Diff line number Diff line
@@ -7,7 +7,7 @@ 
 * License.  See the file "COPYING" in the main directory of this archive

 * for more details.

 *

 * Copyright (C) 2004-2005 by Tensilica Inc.

 * Copyright (C) 2004-2007 by Tensilica Inc.

 *

 * Chris Zankel <chris@zankel.net>

 *
@@ -169,7 +169,7 @@ _user_exception: 
	 * We have to save all registers up to the first '1' from

	 * the right, except the current frame (bit 0).

	 * Assume a2 is:  001001000110001

	 * All regiser frames starting from the top fiel to the marked '1'

	 * All register frames starting from the top field to the marked '1'

	 * must be saved.

	 */
@@ -1590,7 +1590,7 @@ ENTRY(fast_second_level_miss) 
	 * The messy computation for 'pteval' above really simplifies

	 * into the following:

	 *

	 * pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_KERNEL

	 * pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_DIRECTORY

	 */



	movi	a1, -PAGE_OFFSET
@@ -1602,7 +1602,7 @@ ENTRY(fast_second_level_miss) 
	or	a0, a0, a1		# ... | PAGE_DIRECTORY



	/*

	 * We utilize all three wired-ways (7-9( to hold pmd translations.

	 * We utilize all three wired-ways (7-9) to hold pmd translations.

	 * Memory regions are mapped to the DTLBs according to bits 28 and 29.

	 * This allows to map the three most common regions to three different

	 * DTLBs:
@@ -1652,6 +1652,73 @@ ENTRY(fast_second_level_miss) 
9:	l32i	a0, a1, TASK_ACTIVE_MM	# unlikely case mm == 0

	j	8b



#if (DCACHE_WAY_SIZE > PAGE_SIZE)



2:	/* Special case for cache aliasing.

	 * We (should) only get here if a clear_user_page, copy_user_page

	 * or the aliased cache flush functions got preemptively interrupted 

	 * by another task. Re-establish temporary mapping to the 

	 * TLBTEMP_BASE areas.

	 */



	/* We shouldn't be in a double exception */



	l32i	a0, a2, PT_DEPC

	bgeui	a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 2f



	/* Make sure the exception originated in the special functions */



	movi	a0, __tlbtemp_mapping_start

	rsr	a3, EPC_1

	bltu	a3, a0, 2f

	movi	a0, __tlbtemp_mapping_end

	bgeu	a3, a0, 2f



	/* Check if excvaddr was in one of the TLBTEMP_BASE areas. */



	movi	a3, TLBTEMP_BASE_1

	rsr	a0, EXCVADDR

	bltu	a0, a3, 2f



	addi	a1, a0, -(2 << (DCACHE_ALIAS_ORDER + PAGE_SHIFT))

	bgeu	a1, a3, 2f



	/* Check if we have to restore an ITLB mapping. */



	movi	a1, __tlbtemp_mapping_itlb

	rsr	a3, EPC_1

	sub	a3, a3, a1



	/* Calculate VPN */



	movi	a1, PAGE_MASK

	and	a1, a1, a0



	/* Jump for ITLB entry */



	bgez	a3, 1f



	/* We can use up to two TLBTEMP areas, one for src and one for dst. */



	extui	a3, a0, PAGE_SHIFT + DCACHE_ALIAS_ORDER, 1

	add	a1, a3, a1



	/* PPN is in a6 for the first TLBTEMP area and in a7 for the second. */



	mov	a0, a6

	movnez	a0, a7, a3

	j	3b



	/* ITLB entry. We only use dst in a6. */



1:	witlb	a6, a1

	isync

	j	4b





#endif	// DCACHE_WAY_SIZE > PAGE_SIZE





2:	/* Invalid PGD, default exception handling */



	movi	a3, exc_table

arch/xtensa/mm/Makefile

+1 −5
Original line number Diff line number Diff line
@@ -5,9 +5,5 @@ 
# removes any old dependencies. DON'T put your own dependencies here

# unless it's something special (ie not a .c file).

#

# Note 2! The CFLAGS definition is now in the main makefile...



obj-y	 := init.o fault.o tlb.o misc.o

obj-m	 :=

obj-n	 :=

obj-	 :=
 
obj-y	 := init.o fault.o tlb.o misc.o cache.o

arch/xtensa/mm/cache.c

0 → 100644
+256 −0
Original line number Diff line number Diff line 
/*

 * arch/xtensa/mm/cache.c

 *

 * This file is subject to the terms and conditions of the GNU General Public

 * License.  See the file "COPYING" in the main directory of this archive

 * for more details.

 *

 * Copyright (C) 2001-2006 Tensilica Inc.

 *

 * Chris Zankel	<chris@zankel.net>

 * Joe Taylor

 * Marc Gauthier

 *

 */



#include <linux/init.h>

#include <linux/signal.h>

#include <linux/sched.h>

#include <linux/kernel.h>

#include <linux/errno.h>

#include <linux/string.h>

#include <linux/types.h>

#include <linux/ptrace.h>

#include <linux/bootmem.h>

#include <linux/swap.h>

#include <linux/pagemap.h>



#include <asm/pgtable.h>

#include <asm/bootparam.h>

#include <asm/mmu_context.h>

#include <asm/tlb.h>

#include <asm/tlbflush.h>

#include <asm/page.h>

#include <asm/pgalloc.h>

#include <asm/pgtable.h>



//#define printd(x...) printk(x)

#define printd(x...) do { } while(0)



/* 

 * Note:

 * The kernel provides one architecture bit PG_arch_1 in the page flags that 

 * can be used for cache coherency.

 *

 * I$-D$ coherency.

 *

 * The Xtensa architecture doesn't keep the instruction cache coherent with

 * the data cache. We use the architecture bit to indicate if the caches

 * are coherent. The kernel clears this bit whenever a page is added to the

 * page cache. At that time, the caches might not be in sync. We, therefore,

 * define this flag as 'clean' if set.

 *

 * D-cache aliasing.

 *

 * With cache aliasing, we have to always flush the cache when pages are

 * unmapped (see tlb_start_vma(). So, we use this flag to indicate a dirty

 * page.

 * 

 *

 *

 */



#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK



/*

 * Any time the kernel writes to a user page cache page, or it is about to

 * read from a page cache page this routine is called.

 *

 */



void flush_dcache_page(struct page *page)

{

	struct address_space *mapping = page_mapping(page);



	/*

	 * If we have a mapping but the page is not mapped to user-space

	 * yet, we simply mark this page dirty and defer flushing the 

	 * caches until update_mmu().

	 */



	if (mapping && !mapping_mapped(mapping)) {

		if (!test_bit(PG_arch_1, &page->flags))

			set_bit(PG_arch_1, &page->flags);

		return;



	} else {



		unsigned long phys = page_to_phys(page);

		unsigned long temp = page->index << PAGE_SHIFT;

		unsigned long alias = !(DCACHE_ALIAS_EQ(temp, phys));

		unsigned long virt;



		/* 

		 * Flush the page in kernel space and user space.

		 * Note that we can omit that step if aliasing is not

		 * an issue, but we do have to synchronize I$ and D$

		 * if we have a mapping.

		 */



		if (!alias && !mapping)

			return;



		__flush_invalidate_dcache_page((long)page_address(page));



		virt = TLBTEMP_BASE_1 + (temp & DCACHE_ALIAS_MASK);



		if (alias)

			__flush_invalidate_dcache_page_alias(virt, phys);



		if (mapping)

			__invalidate_icache_page_alias(virt, phys);

	}



	/* There shouldn't be an entry in the cache for this page anymore. */

}





/*

 * For now, flush the whole cache. FIXME??

 */



void flush_cache_range(struct vm_area_struct* vma, 

		       unsigned long start, unsigned long end)

{

	__flush_invalidate_dcache_all();

	__invalidate_icache_all();

}



/* 

 * Remove any entry in the cache for this page. 

 *

 * Note that this function is only called for user pages, so use the

 * alias versions of the cache flush functions.

 */



void flush_cache_page(struct vm_area_struct* vma, unsigned long address,

    		      unsigned long pfn)

{

	/* Note that we have to use the 'alias' address to avoid multi-hit */



	unsigned long phys = page_to_phys(pfn_to_page(pfn));

	unsigned long virt = TLBTEMP_BASE_1 + (address & DCACHE_ALIAS_MASK);



	__flush_invalidate_dcache_page_alias(virt, phys);

	__invalidate_icache_page_alias(virt, phys);

}



#endif



void

update_mmu_cache(struct vm_area_struct * vma, unsigned long addr, pte_t pte)

{

	unsigned long pfn = pte_pfn(pte);

	struct page *page;



	if (!pfn_valid(pfn))

		return;



	page = pfn_to_page(pfn);



	/* Invalidate old entry in TLBs */



	invalidate_itlb_mapping(addr);

	invalidate_dtlb_mapping(addr);



#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK



	if (!PageReserved(page) && test_bit(PG_arch_1, &page->flags)) {



		unsigned long vaddr = TLBTEMP_BASE_1 + (addr & DCACHE_ALIAS_MASK);

		unsigned long paddr = (unsigned long) page_address(page);

		unsigned long phys = page_to_phys(page);



		__flush_invalidate_dcache_page(paddr);



		__flush_invalidate_dcache_page_alias(vaddr, phys);

		__invalidate_icache_page_alias(vaddr, phys);



		clear_bit(PG_arch_1, &page->flags);

	}

#else

	if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags)

	    && (vma->vm_flags & VM_EXEC) != 0) {

		unsigned long vaddr = addr & PAGE_MASK;

		__flush_dcache_page(vaddr);

		__invalidate_icache_page(vaddr);

		set_bit(PG_arch_1, &page->flags);

	}

#endif

}



/*

 * access_process_vm() has called get_user_pages(), which has done a

 * flush_dcache_page() on the page.

 */



#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK



void copy_to_user_page(struct vm_area_struct *vma, struct page *page, 

		unsigned long vaddr, void *dst, const void *src,

		unsigned long len)

{

	unsigned long phys = page_to_phys(page);

	unsigned long alias = !(DCACHE_ALIAS_EQ(vaddr, phys));



	/* Flush and invalidate user page if aliased. */



	if (alias) {

		unsigned long temp = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK);

		__flush_invalidate_dcache_page_alias(temp, phys);

	}



	/* Copy data */

	

	memcpy(dst, src, len);



	/*

	 * Flush and invalidate kernel page if aliased and synchronize 

	 * data and instruction caches for executable pages. 

	 */



	if (alias) {

		unsigned long temp = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK);



		__flush_invalidate_dcache_range((unsigned long) dst, len);

		if ((vma->vm_flags & VM_EXEC) != 0) {

			__invalidate_icache_page_alias(temp, phys);

		}



	} else if ((vma->vm_flags & VM_EXEC) != 0) {

		__flush_dcache_range((unsigned long)dst,len);

		__invalidate_icache_range((unsigned long) dst, len);

	}

}



extern void copy_from_user_page(struct vm_area_struct *vma, struct page *page,

		unsigned long vaddr, void *dst, const void *src,

		unsigned long len)

{

	unsigned long phys = page_to_phys(page);

	unsigned long alias = !(DCACHE_ALIAS_EQ(vaddr, phys));



	/*

	 * Flush user page if aliased. 

	 * (Note: a simply flush would be sufficient) 

	 */



	if (alias) {

		unsigned long temp = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK);

		__flush_invalidate_dcache_page_alias(temp, phys);

	}



	memcpy(dst, src, len);

}



#endif

arch/xtensa/mm/fault.c

+4 −2
Original line number Diff line number Diff line
@@ -24,6 +24,8 @@ 
unsigned long asid_cache = ASID_USER_FIRST;

void bad_page_fault(struct pt_regs*, unsigned long, int);



#undef DEBUG_PAGE_FAULT



/*

 * This routine handles page faults.  It determines the address,

 * and the problem, and then passes it off to one of the appropriate
@@ -64,7 +66,7 @@ void do_page_fault(struct pt_regs *regs) 
		    exccause == EXCCAUSE_ITLB_MISS ||

		    exccause == EXCCAUSE_FETCH_CACHE_ATTRIBUTE) ? 1 : 0;



#if 0

#ifdef DEBUG_PAGE_FAULT

	printk("[%s:%d:%08x:%d:%08x:%s%s]\n", current->comm, current->pid,

	       address, exccause, regs->pc, is_write? "w":"", is_exec? "x":"");

#endif
@@ -219,7 +221,7 @@ bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) 


	/* Are we prepared to handle this kernel fault?  */

	if ((entry = search_exception_tables(regs->pc)) != NULL) {

#if 1

#ifdef DEBUG_PAGE_FAULT

		printk(KERN_DEBUG "%s: Exception at pc=%#010lx (%lx)\n",

				current->comm, regs->pc, entry->fixup);

#endif