[PATCH] xtensa: Architecture support for Tensilica Xtensa Part 5

#

# Makefile for the Linux/Xtensa-specific parts of the memory manager.

#

# Note! Dependencies are done automagically by 'make dep', which also

# removes any old dependencies. DON'T put your own dependencies here

# unless it's something special (ie not a .c file).

#

# Note 2! The CFLAGS definition is now in the main makefile...

obj-y	 := init.o fault.o tlb.o misc.o

obj-m	 :=

obj-n	 :=

obj-	 :=

// TODO VM_EXEC flag work-around, cache aliasing

/*

 * arch/xtensa/mm/fault.c

 *

 * This file is subject to the terms and conditions of the GNU General Public

 * License.  See the file "COPYING" in the main directory of this archive

 * for more details.

 *

 * Copyright (C) 2001 - 2005 Tensilica Inc.

 *

 * Chris Zankel <chris@zankel.net>

 * Joe Taylor	<joe@tensilica.com, joetylr@yahoo.com>

 */

#include <linux/mm.h>

#include <linux/module.h>

#include <asm/mmu_context.h>

#include <asm/cacheflush.h>

#include <asm/hardirq.h>

#include <asm/uaccess.h>

#include <asm/system.h>

#include <asm/pgalloc.h>

unsigned long asid_cache = ASID_FIRST_VERSION;

void bad_page_fault(struct pt_regs*, unsigned long, int);

/*

 * This routine handles page faults.  It determines the address,

 * and the problem, and then passes it off to one of the appropriate

 * routines.

 *

 * Note: does not handle Miss and MultiHit.

 */

void do_page_fault(struct pt_regs *regs)

{

	struct vm_area_struct * vma;

	struct mm_struct *mm = current->mm;

	unsigned int exccause = regs->exccause;

	unsigned int address = regs->excvaddr;

	siginfo_t info;

	int is_write, is_exec;

	info.si_code = SEGV_MAPERR;

	/* We fault-in kernel-space virtual memory on-demand. The

	 * 'reference' page table is init_mm.pgd.

	 */

	if (address >= TASK_SIZE && !user_mode(regs))

		goto vmalloc_fault;

	/* If we're in an interrupt or have no user

	 * context, we must not take the fault..

	 */

	if (in_atomic() || !mm) {

		bad_page_fault(regs, address, SIGSEGV);

		return;

	}

	is_write = (exccause == XCHAL_EXCCAUSE_STORE_CACHE_ATTRIBUTE) ? 1 : 0;

	is_exec =  (exccause == XCHAL_EXCCAUSE_ITLB_PRIVILEGE ||

		    exccause == XCHAL_EXCCAUSE_ITLB_MISS ||

		    exccause == XCHAL_EXCCAUSE_FETCH_CACHE_ATTRIBUTE) ? 1 : 0;

#if 0

	printk("[%s:%d:%08x:%d:%08x:%s%s]\n", current->comm, current->pid,

	       address, exccause, regs->pc, is_write? "w":"", is_exec? "x":"");

#endif

	down_read(&mm->mmap_sem);

	vma = find_vma(mm, address);

	if (!vma)

		goto bad_area;

	if (vma->vm_start <= address)

		goto good_area;

	if (!(vma->vm_flags & VM_GROWSDOWN))

		goto bad_area;

	if (expand_stack(vma, address))

		goto bad_area;

	/* Ok, we have a good vm_area for this memory access, so

	 * we can handle it..

	 */

good_area:

	info.si_code = SEGV_ACCERR;

	if (is_write) {

		if (!(vma->vm_flags & VM_WRITE))

			goto bad_area;

	} else if (is_exec) {

		if (!(vma->vm_flags & VM_EXEC))

			goto bad_area;

	} else	/* Allow read even from write-only pages. */

		if (!(vma->vm_flags & (VM_READ | VM_WRITE)))

			goto bad_area;

	/* If for any reason at all we couldn't handle the fault,

	 * make sure we exit gracefully rather than endlessly redo

	 * the fault.

	 */

survive:

	switch (handle_mm_fault(mm, vma, address, is_write)) {

	case VM_FAULT_MINOR:

		current->min_flt++;

		break;

	case VM_FAULT_MAJOR:

		current->maj_flt++;

		break;

	case VM_FAULT_SIGBUS:

		goto do_sigbus;

	case VM_FAULT_OOM:

		goto out_of_memory;

	default:

		BUG();

	}

	up_read(&mm->mmap_sem);

	return;

	/* Something tried to access memory that isn't in our memory map..

	 * Fix it, but check if it's kernel or user first..

	 */

bad_area:

	up_read(&mm->mmap_sem);

	if (user_mode(regs)) {

		current->thread.bad_vaddr = address;

		current->thread.error_code = is_write;

		info.si_signo = SIGSEGV;

		info.si_errno = 0;

		/* info.si_code has been set above */

		info.si_addr = (void *) address;

		force_sig_info(SIGSEGV, &info, current);

		return;

	}

	bad_page_fault(regs, address, SIGSEGV);

	return;

	/* We ran out of memory, or some other thing happened to us that made

	 * us unable to handle the page fault gracefully.

	 */

out_of_memory:

	up_read(&mm->mmap_sem);

	if (current->pid == 1) {

		yield();

		down_read(&mm->mmap_sem);

		goto survive;

	}

	printk("VM: killing process %s\n", current->comm);

	if (user_mode(regs))

		do_exit(SIGKILL);

	bad_page_fault(regs, address, SIGKILL);

	return;

do_sigbus:

	up_read(&mm->mmap_sem);

	/* Send a sigbus, regardless of whether we were in kernel

	 * or user mode.

	 */

	current->thread.bad_vaddr = address;

	info.si_code = SIGBUS;

	info.si_errno = 0;

	info.si_code = BUS_ADRERR;

	info.si_addr = (void *) address;

	force_sig_info(SIGBUS, &info, current);

	/* Kernel mode? Handle exceptions or die */

	if (!user_mode(regs))

		bad_page_fault(regs, address, SIGBUS);

vmalloc_fault:

	{

		/* Synchronize this task's top level page-table

		 * with the 'reference' page table.

		 */

		struct mm_struct *act_mm = current->active_mm;

		int index = pgd_index(address);

		pgd_t *pgd, *pgd_k;

		pmd_t *pmd, *pmd_k;

		pte_t *pte_k;

		if (act_mm == NULL)

			goto bad_page_fault;

		pgd = act_mm->pgd + index;

		pgd_k = init_mm.pgd + index;

		if (!pgd_present(*pgd_k))

			goto bad_page_fault;

		pgd_val(*pgd) = pgd_val(*pgd_k);

		pmd = pmd_offset(pgd, address);

		pmd_k = pmd_offset(pgd_k, address);

		if (!pmd_present(*pmd) || !pmd_present(*pmd_k))

			goto bad_page_fault;

		pmd_val(*pmd) = pmd_val(*pmd_k);

		pte_k = pte_offset_kernel(pmd_k, address);

		if (!pte_present(*pte_k))

			goto bad_page_fault;

		return;

	}

bad_page_fault:

	bad_page_fault(regs, address, SIGKILL);

	return;

}

void

bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)

{

	extern void die(const char*, struct pt_regs*, long);

	const struct exception_table_entry *entry;

	/* Are we prepared to handle this kernel fault?  */

	if ((entry = search_exception_tables(regs->pc)) != NULL) {

#if 1

		printk(KERN_DEBUG "%s: Exception at pc=%#010lx (%lx)\n",

				current->comm, regs->pc, entry->fixup);

#endif

		current->thread.bad_uaddr = address;

		regs->pc = entry->fixup;

		return;

	}

	/* Oops. The kernel tried to access some bad page. We'll have to

	 * terminate things with extreme prejudice.

	 */

	printk(KERN_ALERT "Unable to handle kernel paging request at virtual "

	       "address %08lx\n pc = %08lx, ra = %08lx\n",

	       address, regs->pc, regs->areg[0]);

	die("Oops", regs, sig);

	do_exit(sig);

}

/*

 * arch/xtensa/mm/init.c

 *

 * Derived from MIPS, PPC.

 *

 * This file is subject to the terms and conditions of the GNU General Public

 * License.  See the file "COPYING" in the main directory of this archive

 * for more details.

 *

 * Copyright (C) 2001 - 2005 Tensilica Inc.

 *

 * Chris Zankel	<chris@zankel.net>

 * Joe Taylor	<joe@tensilica.com, joetylr@yahoo.com>

 * Marc Gauthier

 * Kevin Chea

 */

#include <linux/config.h>

#include <linux/init.h>

#include <linux/signal.h>

#include <linux/sched.h>

#include <linux/kernel.h>

#include <linux/errno.h>

#include <linux/string.h>

#include <linux/types.h>

#include <linux/ptrace.h>

#include <linux/bootmem.h>

#include <linux/swap.h>

#include <asm/pgtable.h>

#include <asm/bootparam.h>

#include <asm/mmu_context.h>

#include <asm/tlb.h>

#include <asm/tlbflush.h>

#include <asm/page.h>

#include <asm/pgalloc.h>

#include <asm/pgtable.h>

#define DEBUG 0

DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);

//static DEFINE_SPINLOCK(tlb_lock);

/*

 * This flag is used to indicate that the page was mapped and modified in

 * kernel space, so the cache is probably dirty at that address.

 * If cache aliasing is enabled and the page color mismatches, update_mmu_cache

 * synchronizes the caches if this bit is set.

 */

#define PG_cache_clean PG_arch_1

/* References to section boundaries */

extern char _ftext, _etext, _fdata, _edata, _rodata_end;

extern char __init_begin, __init_end;

/*

 * mem_reserve(start, end, must_exist)

 *

 * Reserve some memory from the memory pool.

 *

 * Parameters:

 *  start	Start of region,

 *  end		End of region,

 *  must_exist	Must exist in memory pool.

 *

 * Returns:

 *  0 (memory area couldn't be mapped)

 * -1 (success)

 */

int __init mem_reserve(unsigned long start, unsigned long end, int must_exist)

{

	int i;

	if (start == end)

		return 0;

	start = start & PAGE_MASK;

	end = PAGE_ALIGN(end);

	for (i = 0; i < sysmem.nr_banks; i++)

		if (start < sysmem.bank[i].end

		    && end >= sysmem.bank[i].start)

			break;

	if (i == sysmem.nr_banks) {

		if (must_exist)

			printk (KERN_WARNING "mem_reserve: [0x%0lx, 0x%0lx) "

				"not in any region!\n", start, end);

		return 0;

	}

	if (start > sysmem.bank[i].start) {

		if (end < sysmem.bank[i].end) {

			/* split entry */

			if (sysmem.nr_banks >= SYSMEM_BANKS_MAX)

				panic("meminfo overflow\n");

			sysmem.bank[sysmem.nr_banks].start = end;

			sysmem.bank[sysmem.nr_banks].end = sysmem.bank[i].end;

			sysmem.nr_banks++;

		}

		sysmem.bank[i].end = start;

	} else {

		if (end < sysmem.bank[i].end)

			sysmem.bank[i].start = end;

		else {

			/* remove entry */

			sysmem.nr_banks--;

			sysmem.bank[i].start = sysmem.bank[sysmem.nr_banks].start;

			sysmem.bank[i].end   = sysmem.bank[sysmem.nr_banks].end;

		}

	}

	return -1;

}

/*

 * Initialize the bootmem system and give it all the memory we have available.

 */

void __init bootmem_init(void)

{

	unsigned long pfn;

	unsigned long bootmap_start, bootmap_size;

	int i;

	max_low_pfn = max_pfn = 0;

	min_low_pfn = ~0;

	for (i=0; i < sysmem.nr_banks; i++) {

		pfn = PAGE_ALIGN(sysmem.bank[i].start) >> PAGE_SHIFT;

		if (pfn < min_low_pfn)

			min_low_pfn = pfn;

		pfn = PAGE_ALIGN(sysmem.bank[i].end - 1) >> PAGE_SHIFT;

		if (pfn > max_pfn)

			max_pfn = pfn;

	}

	if (min_low_pfn > max_pfn)

		panic("No memory found!\n");

	max_low_pfn = max_pfn < MAX_LOW_MEMORY >> PAGE_SHIFT ?

		max_pfn : MAX_LOW_MEMORY >> PAGE_SHIFT;

	/* Find an area to use for the bootmem bitmap. */

	bootmap_size = bootmem_bootmap_pages(max_low_pfn) << PAGE_SHIFT;

	bootmap_start = ~0;

	for (i=0; i<sysmem.nr_banks; i++)

		if (sysmem.bank[i].end - sysmem.bank[i].start >= bootmap_size) {

			bootmap_start = sysmem.bank[i].start;

			break;

		}

	if (bootmap_start == ~0UL)

		panic("Cannot find %ld bytes for bootmap\n", bootmap_size);

	/* Reserve the bootmem bitmap area */

	mem_reserve(bootmap_start, bootmap_start + bootmap_size, 1);

	bootmap_size = init_bootmem_node(NODE_DATA(0), min_low_pfn,

					 bootmap_start >> PAGE_SHIFT,

					 max_low_pfn);

	/* Add all remaining memory pieces into the bootmem map */

	for (i=0; i<sysmem.nr_banks; i++)

		free_bootmem(sysmem.bank[i].start,

			     sysmem.bank[i].end - sysmem.bank[i].start);

}

void __init paging_init(void)

{

	unsigned long zones_size[MAX_NR_ZONES];

	int i;

	/* All pages are DMA-able, so we put them all in the DMA zone. */

	zones_size[ZONE_DMA] = max_low_pfn;

	for (i = 1; i < MAX_NR_ZONES; i++)

		zones_size[i] = 0;

#ifdef CONFIG_HIGHMEM

	zones_size[ZONE_HIGHMEM] = max_pfn - max_low_pfn;

#endif

	/* Initialize the kernel's page tables. */

	memset(swapper_pg_dir, 0, PAGE_SIZE);

	free_area_init(zones_size);

}

/*

 * Flush the mmu and reset associated register to default values.

 */

void __init init_mmu (void)

{

	/* Writing zeros to the <t>TLBCFG special registers ensure

	 * that valid values exist in the register.  For existing

	 * PGSZID<w> fields, zero selects the first element of the

	 * page-size array.  For nonexistant PGSZID<w> fields, zero is

	 * the best value to write.  Also, when changing PGSZID<w>

	 * fields, the corresponding TLB must be flushed.

	 */

	set_itlbcfg_register (0);

	set_dtlbcfg_register (0);

	flush_tlb_all ();

	/* Set rasid register to a known value. */

	set_rasid_register (ASID_ALL_RESERVED);

	/* Set PTEVADDR special register to the start of the page

	 * table, which is in kernel mappable space (ie. not

	 * statically mapped).  This register's value is undefined on

	 * reset.

	 */

	set_ptevaddr_register (PGTABLE_START);

}

/*

 * Initialize memory pages.

 */

void __init mem_init(void)

{

	unsigned long codesize, reservedpages, datasize, initsize;

	unsigned long highmemsize, tmp, ram;

	max_mapnr = num_physpages = max_low_pfn;

	high_memory = (void *) __va(max_mapnr << PAGE_SHIFT);

	highmemsize = 0;

#if CONFIG_HIGHMEM

#error HIGHGMEM not implemented in init.c

#endif

	totalram_pages += free_all_bootmem();

	reservedpages = ram = 0;

	for (tmp = 0; tmp < max_low_pfn; tmp++) {

		ram++;

		if (PageReserved(mem_map+tmp))

			reservedpages++;

	}

	codesize =  (unsigned long) &_etext - (unsigned long) &_ftext;

	datasize =  (unsigned long) &_edata - (unsigned long) &_fdata;

	initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;

	printk("Memory: %luk/%luk available (%ldk kernel code, %ldk reserved, "

	       "%ldk data, %ldk init %ldk highmem)\n",

	       (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),

	       ram << (PAGE_SHIFT-10),

	       codesize >> 10,

	       reservedpages << (PAGE_SHIFT-10),

	       datasize >> 10,

	       initsize >> 10,

	       highmemsize >> 10);

}

void

free_reserved_mem(void *start, void *end)

{

	for (; start < end; start += PAGE_SIZE) {

		ClearPageReserved(virt_to_page(start));

		set_page_count(virt_to_page(start), 1);

		free_page((unsigned long)start);

		totalram_pages++;

	}

}

#ifdef CONFIG_BLK_DEV_INITRD

extern int initrd_is_mapped;

void free_initrd_mem(unsigned long start, unsigned long end)

{

	if (initrd_is_mapped) {

		free_reserved_mem((void*)start, (void*)end);

		printk ("Freeing initrd memory: %ldk freed\n",(end-start)>>10);

	}

}

#endif

void free_initmem(void)

{

	free_reserved_mem(&__init_begin, &__init_end);

	printk("Freeing unused kernel memory: %dk freed\n",

	       (&__init_end - &__init_begin) >> 10);

}

void show_mem(void)

{

	int i, free = 0, total = 0, reserved = 0;

	int shared = 0, cached = 0;

	printk("Mem-info:\n");

	show_free_areas();

	printk("Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));

	i = max_mapnr;

	while (i-- > 0) {

		total++;

		if (PageReserved(mem_map+i))

			reserved++;

		else if (PageSwapCache(mem_map+i))

			cached++;

		else if (!page_count(mem_map + i))

			free++;

		else

			shared += page_count(mem_map + i) - 1;

	}

	printk("%d pages of RAM\n", total);

	printk("%d reserved pages\n", reserved);

	printk("%d pages shared\n", shared);

	printk("%d pages swap cached\n",cached);

	printk("%d free pages\n", free);

}

/* ------------------------------------------------------------------------- */

#if (DCACHE_WAY_SIZE > PAGE_SIZE)

/*

 * With cache aliasing, the page color of the page in kernel space and user

 * space might mismatch. We temporarily map the page to a different virtual

 * address with the same color and clear the page there.

 */

void clear_user_page(void *kaddr, unsigned long vaddr, struct page* page)

{

  	/*  There shouldn't be any entries for this page. */

	__flush_invalidate_dcache_page_phys(__pa(page_address(page)));

	if (!PAGE_COLOR_EQ(vaddr, kaddr)) {

		unsigned long v, p;

		/* Temporarily map page to DTLB_WAY_DCACHE_ALIAS0. */

		spin_lock(&tlb_lock);

		p = (unsigned long)pte_val((mk_pte(page,PAGE_KERNEL)));

		kaddr = (void*)PAGE_COLOR_MAP0(vaddr);

		v = (unsigned long)kaddr | DTLB_WAY_DCACHE_ALIAS0;

		__asm__ __volatile__("wdtlb %0,%1; dsync" : :"a" (p), "a" (v));

		clear_page(kaddr);

		spin_unlock(&tlb_lock);

	} else {

		clear_page(kaddr);

	}

	/* We need to make sure that i$ and d$ are coherent. */

	clear_bit(PG_cache_clean, &page->flags);

}

/*

 * With cache aliasing, we have to make sure that the page color of the page

 * in kernel space matches that of the virtual user address before we read

 * the page. If the page color differ, we create a temporary DTLB entry with

 * the corrent page color and use this 'temporary' address as the source.

 * We then use the same approach as in clear_user_page and copy the data

 * to the kernel space and clear the PG_cache_clean bit to synchronize caches

 * later.

 *

 * Note:

 * Instead of using another 'way' for the temporary DTLB entry, we could

 * probably use the same entry that points to the kernel address (after

 * saving the original value and restoring it when we are done).

 */

void copy_user_page(void* to, void* from, unsigned long vaddr,

    		    struct page* to_page)

{

	/* There shouldn't be any entries for the new page. */

	__flush_invalidate_dcache_page_phys(__pa(page_address(to_page)));

	spin_lock(&tlb_lock);

	if (!PAGE_COLOR_EQ(vaddr, from)) {

		unsigned long v, p, t;

		__asm__ __volatile__ ("pdtlb %1,%2; rdtlb1 %0,%1"

				      : "=a"(p), "=a"(t) : "a"(from));

		from = (void*)PAGE_COLOR_MAP0(vaddr);

		v = (unsigned long)from | DTLB_WAY_DCACHE_ALIAS0;

		__asm__ __volatile__ ("wdtlb %0,%1; dsync" ::"a" (p), "a" (v));

	}

	if (!PAGE_COLOR_EQ(vaddr, to)) {

		unsigned long v, p;

		p = (unsigned long)pte_val((mk_pte(to_page,PAGE_KERNEL)));

		to = (void*)PAGE_COLOR_MAP1(vaddr);

		v = (unsigned long)to | DTLB_WAY_DCACHE_ALIAS1;

		__asm__ __volatile__ ("wdtlb %0,%1; dsync" ::"a" (p), "a" (v));

	}

	copy_page(to, from);

	spin_unlock(&tlb_lock);

	/* We need to make sure that i$ and d$ are coherent. */

	clear_bit(PG_cache_clean, &to_page->flags);

}

/*

 * Any time the kernel writes to a user page cache page, or it is about to

 * read from a page cache page this routine is called.

 *

 * Note:

 * The kernel currently only provides one architecture bit in the page

 * flags that we use for I$/D$ coherency. Maybe, in future, we can

 * use a sepearte bit for deferred dcache aliasing:

 * If the page is not mapped yet, we only need to set a flag,

 * if mapped, we need to invalidate the page.

 */

// FIXME: we probably need this for WB caches not only for Page Coloring..

void flush_dcache_page(struct page *page)

{

	unsigned long addr = __pa(page_address(page));

	struct address_space *mapping = page_mapping(page);

	__flush_invalidate_dcache_page_phys(addr);

	if (!test_bit(PG_cache_clean, &page->flags))

		return;

	/* If this page hasn't been mapped, yet, handle I$/D$ coherency later.*/

#if 0

	if (mapping && !mapping_mapped(mapping))

		clear_bit(PG_cache_clean, &page->flags);

	else

#endif

		__invalidate_icache_page_phys(addr);

}

void flush_cache_range(struct vm_area_struct* vma, unsigned long s,

		       unsigned long e)

{

	__flush_invalidate_cache_all();

}

void flush_cache_page(struct vm_area_struct* vma, unsigned long address,

    		      unsigned long pfn)

{

	struct page *page = pfn_to_page(pfn);

	/* Remove any entry for the old mapping. */

	if (current->active_mm == vma->vm_mm) {

		unsigned long addr = __pa(page_address(page));

		__flush_invalidate_dcache_page_phys(addr);

		if ((vma->vm_flags & VM_EXEC) != 0)

			__invalidate_icache_page_phys(addr);

	} else {

		BUG();

	}

}

#endif	/* (DCACHE_WAY_SIZE > PAGE_SIZE) */

pte_t* pte_alloc_one_kernel (struct mm_struct* mm, unsigned long addr)