Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a0ad5496 authored by Steve Capper's avatar Steve Capper Committed by Linus Torvalds
Browse files

arm: mm: enable HAVE_RCU_TABLE_FREE logic 



In order to implement fast_get_user_pages we need to ensure that the page
table walker is protected from page table pages being freed from under it.

This patch enables HAVE_RCU_TABLE_FREE, any page table pages belonging to
address spaces with multiple users will be call_rcu_sched freed.  Meaning
that disabling interrupts will block the free and protect the fast gup
page walker.

Signed-off-by: default avatarSteve Capper <steve.capper@linaro.org>
Reviewed-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
Cc: Dann Frazier <dann.frazier@canonical.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Russell King <rmk@arm.linux.org.uk>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent bd951303

arch/arm/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -62,6 +62,7 @@ config ARM 
	select HAVE_PERF_EVENTS

	select HAVE_PERF_REGS

	select HAVE_PERF_USER_STACK_DUMP

	select HAVE_RCU_TABLE_FREE if (SMP && ARM_LPAE)

	select HAVE_REGS_AND_STACK_ACCESS_API

	select HAVE_SYSCALL_TRACEPOINTS

	select HAVE_UID16

arch/arm/include/asm/tlb.h

+36 −2
Original line number Diff line number Diff line
@@ -35,12 +35,39 @@ 


#define MMU_GATHER_BUNDLE	8



#ifdef CONFIG_HAVE_RCU_TABLE_FREE

static inline void __tlb_remove_table(void *_table)

{

	free_page_and_swap_cache((struct page *)_table);

}



struct mmu_table_batch {

	struct rcu_head		rcu;

	unsigned int		nr;

	void			*tables[0];

};



#define MAX_TABLE_BATCH		\

	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))



extern void tlb_table_flush(struct mmu_gather *tlb);

extern void tlb_remove_table(struct mmu_gather *tlb, void *table);



#define tlb_remove_entry(tlb, entry)	tlb_remove_table(tlb, entry)

#else

#define tlb_remove_entry(tlb, entry)	tlb_remove_page(tlb, entry)

#endif /* CONFIG_HAVE_RCU_TABLE_FREE */



/*

 * TLB handling.  This allows us to remove pages from the page

 * tables, and efficiently handle the TLB issues.

 */

struct mmu_gather {

	struct mm_struct	*mm;

#ifdef CONFIG_HAVE_RCU_TABLE_FREE

	struct mmu_table_batch	*batch;

	unsigned int		need_flush;

#endif

	unsigned int		fullmm;

	struct vm_area_struct	*vma;

	unsigned long		start, end;
@@ -101,6 +128,9 @@ static inline void __tlb_alloc_page(struct mmu_gather *tlb) 
static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)

{

	tlb_flush(tlb);

#ifdef CONFIG_HAVE_RCU_TABLE_FREE

	tlb_table_flush(tlb);

#endif

}



static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
@@ -129,6 +159,10 @@ tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start 
	tlb->pages = tlb->local;

	tlb->nr = 0;

	__tlb_alloc_page(tlb);



#ifdef CONFIG_HAVE_RCU_TABLE_FREE

	tlb->batch = NULL;

#endif

}



static inline void
@@ -205,7 +239,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, 
	tlb_add_flush(tlb, addr + SZ_1M);

#endif



	tlb_remove_page(tlb, pte);

	tlb_remove_entry(tlb, pte);

}



static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
@@ -213,7 +247,7 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, 
{

#ifdef CONFIG_ARM_LPAE

	tlb_add_flush(tlb, addr);

	tlb_remove_page(tlb, virt_to_page(pmdp));

	tlb_remove_entry(tlb, virt_to_page(pmdp));

#endif

}