Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 26723911 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Linus Torvalds
Browse files

mm, powerpc: move the RCU page-table freeing into generic code 



In case other architectures require RCU freed page-tables to implement
gup_fast() and software filled hashes and similar things, provide the
means to do so by moving the logic into generic code.

Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Requested-by: default avatarDavid Miller <davem@davemloft.net>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Russell King <rmk@arm.linux.org.uk>
Cc: Paul Mundt <lethal@linux-sh.org>
Cc: Jeff Dike <jdike@addtoit.com>
Cc: Richard Weinberger <richard@nod.at>
Cc: Tony Luck <tony.luck@intel.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Nick Piggin <npiggin@kernel.dk>
Cc: Namhyung Kim <namhyung@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 1c395176

arch/Kconfig

+3 −0
Original line number Diff line number Diff line
@@ -175,4 +175,7 @@ config HAVE_ARCH_JUMP_LABEL 
config HAVE_ARCH_MUTEX_CPU_RELAX

	bool



config HAVE_RCU_TABLE_FREE

	bool



source "kernel/gcov/Kconfig"

arch/powerpc/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -140,6 +140,7 @@ config PPC 
	select IRQ_PER_CPU

	select GENERIC_IRQ_SHOW

	select GENERIC_IRQ_SHOW_LEVEL

	select HAVE_RCU_TABLE_FREE if SMP



config EARLY_PRINTK

	bool

arch/powerpc/include/asm/pgalloc.h

+18 −3
Original line number Diff line number Diff line
@@ -31,14 +31,29 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) 
#endif



#ifdef CONFIG_SMP

extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, unsigned shift);

extern void pte_free_finish(struct mmu_gather *tlb);

struct mmu_gather;

extern void tlb_remove_table(struct mmu_gather *, void *);



static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)

{

	unsigned long pgf = (unsigned long)table;

	BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);

	pgf |= shift;

	tlb_remove_table(tlb, (void *)pgf);

}



static inline void __tlb_remove_table(void *_table)

{

	void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);

	unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;



	pgtable_free(table, shift);

}

#else /* CONFIG_SMP */

static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, unsigned shift)

{

	pgtable_free(table, shift);

}

static inline void pte_free_finish(struct mmu_gather *tlb) { }

#endif /* !CONFIG_SMP */



static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage,

arch/powerpc/include/asm/tlb.h

+0 −10
Original line number Diff line number Diff line
@@ -28,16 +28,6 @@ 
#define tlb_start_vma(tlb, vma)	do { } while (0)

#define tlb_end_vma(tlb, vma)	do { } while (0)



#define HAVE_ARCH_MMU_GATHER 1



struct pte_freelist_batch;



struct arch_mmu_gather {

	struct pte_freelist_batch *batch;

};



#define ARCH_MMU_GATHER_INIT (struct arch_mmu_gather){ .batch = NULL, }



extern void tlb_flush(struct mmu_gather *tlb);



/* Get the generic bits... */

arch/powerpc/mm/pgtable.c

+0 −98
Original line number Diff line number Diff line
@@ -33,104 +33,6 @@ 


#include "mmu_decl.h"



#ifdef CONFIG_SMP



/*

 * Handle batching of page table freeing on SMP. Page tables are

 * queued up and send to be freed later by RCU in order to avoid

 * freeing a page table page that is being walked without locks

 */



static unsigned long pte_freelist_forced_free;



struct pte_freelist_batch

{

	struct rcu_head	rcu;

	unsigned int	index;

	unsigned long	tables[0];

};



#define PTE_FREELIST_SIZE \

	((PAGE_SIZE - sizeof(struct pte_freelist_batch)) \

	  / sizeof(unsigned long))



static void pte_free_smp_sync(void *arg)

{

	/* Do nothing, just ensure we sync with all CPUs */

}



/* This is only called when we are critically out of memory

 * (and fail to get a page in pte_free_tlb).

 */

static void pgtable_free_now(void *table, unsigned shift)

{

	pte_freelist_forced_free++;



	smp_call_function(pte_free_smp_sync, NULL, 1);



	pgtable_free(table, shift);

}



static void pte_free_rcu_callback(struct rcu_head *head)

{

	struct pte_freelist_batch *batch =

		container_of(head, struct pte_freelist_batch, rcu);

	unsigned int i;



	for (i = 0; i < batch->index; i++) {

		void *table = (void *)(batch->tables[i] & ~MAX_PGTABLE_INDEX_SIZE);

		unsigned shift = batch->tables[i] & MAX_PGTABLE_INDEX_SIZE;



		pgtable_free(table, shift);

	}



	free_page((unsigned long)batch);

}



static void pte_free_submit(struct pte_freelist_batch *batch)

{

	call_rcu_sched(&batch->rcu, pte_free_rcu_callback);

}



void pgtable_free_tlb(struct mmu_gather *tlb, void *table, unsigned shift)

{

	struct pte_freelist_batch **batchp = &tlb->arch.batch;

	unsigned long pgf;



	if (atomic_read(&tlb->mm->mm_users) < 2) {

		pgtable_free(table, shift);

		return;

	}



	if (*batchp == NULL) {

		*batchp = (struct pte_freelist_batch *)__get_free_page(GFP_ATOMIC);

		if (*batchp == NULL) {

			pgtable_free_now(table, shift);

			return;

		}

		(*batchp)->index = 0;

	}

	BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);

	pgf = (unsigned long)table | shift;

	(*batchp)->tables[(*batchp)->index++] = pgf;

	if ((*batchp)->index == PTE_FREELIST_SIZE) {

		pte_free_submit(*batchp);

		*batchp = NULL;

	}

}



void pte_free_finish(struct mmu_gather *tlb)

{

	struct pte_freelist_batch **batchp = &tlb->arch.batch;



	if (*batchp == NULL)

		return;

	pte_free_submit(*batchp);

	*batchp = NULL;

}



#endif /* CONFIG_SMP */



static inline int is_exec_fault(void)

{

	return current->thread.regs && TRAP(current->thread.regs) == 0x400;