Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 08f051ed authored by Andrew Waterman's avatar Andrew Waterman Committed by Palmer Dabbelt
Browse files

RISC-V: Flush I$ when making a dirty page executable 



The RISC-V ISA allows for instruction caches that are not coherent WRT
stores, even on a single hart.  As a result, we need to explicitly flush
the instruction cache whenever marking a dirty page as executable in
order to preserve the correct system behavior.

Local instruction caches aren't that scary (our implementations actually
flush the cache, but RISC-V is defined to allow higher-performance
implementations to exist), but RISC-V defines no way to perform an
instruction cache shootdown.  When explicitly asked to do so we can
shoot down remote instruction caches via an IPI, but this is a bit on
the slow side.

Instead of requiring an IPI to all harts whenever marking a page as
executable, we simply flush the currently running harts.  In order to
maintain correct behavior, we additionally mark every other hart as
needing a deferred instruction cache which will be taken before anything
runs on it.

Signed-off-by: default avatarAndrew Waterman <andrew@sifive.com>
Signed-off-by: default avatarPalmer Dabbelt <palmer@sifive.com>
parent 28dfbe6e

arch/riscv/include/asm/cacheflush.h

+20 −4
Original line number Diff line number Diff line
@@ -18,21 +18,37 @@ 


#undef flush_icache_range

#undef flush_icache_user_range

#undef flush_dcache_page



static inline void local_flush_icache_all(void)

{

	asm volatile ("fence.i" ::: "memory");

}



#define PG_dcache_clean PG_arch_1



static inline void flush_dcache_page(struct page *page)

{

	if (test_bit(PG_dcache_clean, &page->flags))

		clear_bit(PG_dcache_clean, &page->flags);

}



/*

 * RISC-V doesn't have an instruction to flush parts of the instruction cache,

 * so instead we just flush the whole thing.

 */

#define flush_icache_range(start, end) flush_icache_all()

#define flush_icache_user_range(vma, pg, addr, len) flush_icache_all()



#ifndef CONFIG_SMP



#define flush_icache_range(start, end) local_flush_icache_all()

#define flush_icache_user_range(vma, pg, addr, len) local_flush_icache_all()

#define flush_icache_all() local_flush_icache_all()

#define flush_icache_mm(mm, local) flush_icache_all()



#else /* CONFIG_SMP */



#define flush_icache_range(start, end) sbi_remote_fence_i(0)

#define flush_icache_user_range(vma, pg, addr, len) sbi_remote_fence_i(0)

#define flush_icache_all() sbi_remote_fence_i(0)

void flush_icache_mm(struct mm_struct *mm, bool local);



#endif /* CONFIG_SMP */

arch/riscv/include/asm/mmu.h

+4 −0
Original line number Diff line number Diff line
@@ -19,6 +19,10 @@ 


typedef struct {

	void *vdso;

#ifdef CONFIG_SMP

	/* A local icache flush is needed before user execution can resume. */

	cpumask_t icache_stale_mask;

#endif

} mm_context_t;



#endif /* __ASSEMBLY__ */

arch/riscv/include/asm/mmu_context.h

+44 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2012 Regents of the University of California

 * Copyright (C) 2017 SiFive

 *

 *   This program is free software; you can redistribute it and/or

 *   modify it under the terms of the GNU General Public License
@@ -19,6 +20,7 @@ 
#include <linux/mm.h>

#include <linux/sched.h>

#include <asm/tlbflush.h>

#include <asm/cacheflush.h>



static inline void enter_lazy_tlb(struct mm_struct *mm,

	struct task_struct *task)
@@ -46,12 +48,54 @@ static inline void set_pgdir(pgd_t *pgd) 
	csr_write(sptbr, virt_to_pfn(pgd) | SPTBR_MODE);

}



/*

 * When necessary, performs a deferred icache flush for the given MM context,

 * on the local CPU.  RISC-V has no direct mechanism for instruction cache

 * shoot downs, so instead we send an IPI that informs the remote harts they

 * need to flush their local instruction caches.  To avoid pathologically slow

 * behavior in a common case (a bunch of single-hart processes on a many-hart

 * machine, ie 'make -j') we avoid the IPIs for harts that are not currently

 * executing a MM context and instead schedule a deferred local instruction

 * cache flush to be performed before execution resumes on each hart.  This

 * actually performs that local instruction cache flush, which implicitly only

 * refers to the current hart.

 */

static inline void flush_icache_deferred(struct mm_struct *mm)

{

#ifdef CONFIG_SMP

	unsigned int cpu = smp_processor_id();

	cpumask_t *mask = &mm->context.icache_stale_mask;



	if (cpumask_test_cpu(cpu, mask)) {

		cpumask_clear_cpu(cpu, mask);

		/*

		 * Ensure the remote hart's writes are visible to this hart.

		 * This pairs with a barrier in flush_icache_mm.

		 */

		smp_mb();

		local_flush_icache_all();

	}

#endif

}



static inline void switch_mm(struct mm_struct *prev,

	struct mm_struct *next, struct task_struct *task)

{

	if (likely(prev != next)) {

		/*

		 * Mark the current MM context as inactive, and the next as

		 * active.  This is at least used by the icache flushing

		 * routines in order to determine who should

		 */

		unsigned int cpu = smp_processor_id();



		cpumask_clear_cpu(cpu, mm_cpumask(prev));

		cpumask_set_cpu(cpu, mm_cpumask(next));



		set_pgdir(next->pgd);

		local_flush_tlb_all();



		flush_icache_deferred(next);

	}

}

arch/riscv/include/asm/pgtable.h

+32 −26
Original line number Diff line number Diff line
@@ -178,28 +178,6 @@ static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long addr) 
#define pte_offset_map(dir, addr)	pte_offset_kernel((dir), (addr))

#define pte_unmap(pte)			((void)(pte))



/*

 * Certain architectures need to do special things when PTEs within

 * a page table are directly modified.  Thus, the following hook is

 * made available.

 */

static inline void set_pte(pte_t *ptep, pte_t pteval)

{

	*ptep = pteval;

}



static inline void set_pte_at(struct mm_struct *mm,

	unsigned long addr, pte_t *ptep, pte_t pteval)

{

	set_pte(ptep, pteval);

}



static inline void pte_clear(struct mm_struct *mm,

	unsigned long addr, pte_t *ptep)

{

	set_pte_at(mm, addr, ptep, __pte(0));

}



static inline int pte_present(pte_t pte)

{

	return (pte_val(pte) & _PAGE_PRESENT);
@@ -210,21 +188,22 @@ static inline int pte_none(pte_t pte) 
	return (pte_val(pte) == 0);

}



/* static inline int pte_read(pte_t pte) */



static inline int pte_write(pte_t pte)

{

	return pte_val(pte) & _PAGE_WRITE;

}



static inline int pte_exec(pte_t pte)

{

	return pte_val(pte) & _PAGE_EXEC;

}



static inline int pte_huge(pte_t pte)

{

	return pte_present(pte)

		&& (pte_val(pte) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC));

}



/* static inline int pte_exec(pte_t pte) */



static inline int pte_dirty(pte_t pte)

{

	return pte_val(pte) & _PAGE_DIRTY;
@@ -311,6 +290,33 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b) 
	return pte_val(pte_a) == pte_val(pte_b);

}



/*

 * Certain architectures need to do special things when PTEs within

 * a page table are directly modified.  Thus, the following hook is

 * made available.

 */

static inline void set_pte(pte_t *ptep, pte_t pteval)

{

	*ptep = pteval;

}



void flush_icache_pte(pte_t pte);



static inline void set_pte_at(struct mm_struct *mm,

	unsigned long addr, pte_t *ptep, pte_t pteval)

{

	if (pte_present(pteval) && pte_exec(pteval))

		flush_icache_pte(pteval);



	set_pte(ptep, pteval);

}



static inline void pte_clear(struct mm_struct *mm,

	unsigned long addr, pte_t *ptep)

{

	set_pte_at(mm, addr, ptep, __pte(0));

}



#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS

static inline int ptep_set_access_flags(struct vm_area_struct *vma,

					unsigned long address, pte_t *ptep,

arch/riscv/include/asm/tlbflush.h

+2 −0
Original line number Diff line number Diff line
@@ -17,6 +17,8 @@ 


#ifdef CONFIG_MMU



#include <linux/mm_types.h>



/* Flush entire local TLB */

static inline void local_flush_tlb_all(void)

{