Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e009bb30 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds
Browse files

mm: implement split page table lock for PMD level 



The basic idea is the same as with PTE level: the lock is embedded into
struct page of table's page.

We can't use mm->pmd_huge_pte to store pgtables for THP, since we don't
take mm->page_table_lock anymore.  Let's reuse page->lru of table's page
for that.

pgtable_pmd_page_ctor() returns true, if initialization is successful
and false otherwise.  Current implementation never fails, but assumption
that constructor can fail will help to port it to -rt where spinlock_t
is rather huge and cannot be embedded into struct page -- dynamic
allocation is required.

Signed-off-by: default avatarNaoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Tested-by: default avatarAlex Thorlton <athorlton@sgi.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "Eric W . Biederman" <ebiederm@xmission.com>
Cc: "Paul E . McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Dave Jones <davej@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kees Cook <keescook@chromium.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Robin Holt <robinmholt@gmail.com>
Cc: Sedat Dilek <sedat.dilek@gmail.com>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Hugh Dickins <hughd@google.com>
Reviewed-by: default avatarSteven Rostedt <rostedt@goodmis.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent c4088ebd

include/linux/mm.h

+32 −0
Original line number Diff line number Diff line
@@ -1378,13 +1378,45 @@ static inline void pgtable_page_dtor(struct page *page) 
	((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \

		NULL: pte_offset_kernel(pmd, address))



#if USE_SPLIT_PMD_PTLOCKS



static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)

{

	return &virt_to_page(pmd)->ptl;

}



static inline bool pgtable_pmd_page_ctor(struct page *page)

{

	spin_lock_init(&page->ptl);

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

	page->pmd_huge_pte = NULL;

#endif

	return true;

}



static inline void pgtable_pmd_page_dtor(struct page *page)

{

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

	VM_BUG_ON(page->pmd_huge_pte);

#endif

}



#define pmd_huge_pte(mm, pmd) (virt_to_page(pmd)->pmd_huge_pte)



#else



static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)

{

	return &mm->page_table_lock;

}



static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }

static inline void pgtable_pmd_page_dtor(struct page *page) {}



#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)



#endif



static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)

{

	spinlock_t *ptl = pmd_lockptr(mm, pmd);

include/linux/mm_types.h

+6 −1
Original line number Diff line number Diff line
@@ -24,6 +24,8 @@ 
struct address_space;



#define USE_SPLIT_PTE_PTLOCKS	(NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)

#define USE_SPLIT_PMD_PTLOCKS	(USE_SPLIT_PTE_PTLOCKS && \

		IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))



/*

 * Each physical page in the system has a struct page associated with
@@ -63,6 +65,9 @@ struct page { 
						 * this page is only used to

						 * free other pages.

						 */

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS

		pgtable_t pmd_huge_pte; /* protected by page->ptl */

#endif

		};



		union {
@@ -406,7 +411,7 @@ struct mm_struct { 
#ifdef CONFIG_MMU_NOTIFIER

	struct mmu_notifier_mm *mmu_notifier_mm;

#endif

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS

	pgtable_t pmd_huge_pte; /* protected by page_table_lock */

#endif

#ifdef CONFIG_CPUMASK_OFFSTACK

kernel/fork.c

+2 −2
Original line number Diff line number Diff line
@@ -560,7 +560,7 @@ static void check_mm(struct mm_struct *mm) 
					  "mm:%p idx:%d val:%ld\n", mm, i, x);

	}



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS

	VM_BUG_ON(mm->pmd_huge_pte);

#endif

}
@@ -814,7 +814,7 @@ struct mm_struct *dup_mm(struct task_struct *tsk) 
	memcpy(mm, oldmm, sizeof(*mm));

	mm_init_cpumask(mm);



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS

	mm->pmd_huge_pte = NULL;

#endif

	if (!mm_init(mm, tsk))

mm/Kconfig

+3 −0
Original line number Diff line number Diff line
@@ -222,6 +222,9 @@ config SPLIT_PTLOCK_CPUS 
	default "999999" if !64BIT && GENERIC_LOCKBREAK

	default "4"



config ARCH_ENABLE_SPLIT_PMD_PTLOCK

	boolean



#

# support for memory balloon compaction

config BALLOON_COMPACTION