Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 451ea25d authored by Johannes Weiner's avatar Johannes Weiner Committed by Linus Torvalds
Browse files

mm: perform non-atomic test-clear of PG_mlocked on free 



By the time PG_mlocked is cleared in the page freeing path, nobody else is
looking at our page->flags anymore.

It is thus safe to make the test-and-clear non-atomic and thereby removing
an unnecessary and expensive operation from a hotpath.

Signed-off-by: default avatarJohannes Weiner <hannes@cmpxchg.org>
Reviewed-by: default avatarChristoph Lameter <cl@linux-foundation.org>
Reviewed-by: default avatarKOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent bf88c8c8

include/linux/page-flags.h

+9 −3
Original line number Diff line number Diff line
@@ -158,6 +158,9 @@ static inline int TestSetPage##uname(struct page *page) \ 
static inline int TestClearPage##uname(struct page *page)		\

		{ return test_and_clear_bit(PG_##lname, &page->flags); }



#define __TESTCLEARFLAG(uname, lname)					\

static inline int __TestClearPage##uname(struct page *page)		\

		{ return __test_and_clear_bit(PG_##lname, &page->flags); }



#define PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname)		\

	SETPAGEFLAG(uname, lname) CLEARPAGEFLAG(uname, lname)
@@ -184,6 +187,9 @@ static inline void __ClearPage##uname(struct page *page) { } 
#define TESTCLEARFLAG_FALSE(uname)					\

static inline int TestClearPage##uname(struct page *page) { return 0; }



#define __TESTCLEARFLAG_FALSE(uname)					\

static inline int __TestClearPage##uname(struct page *page) { return 0; }



struct page;	/* forward declaration */



TESTPAGEFLAG(Locked, locked) TESTSETFLAG(Locked, locked)
@@ -250,11 +256,11 @@ PAGEFLAG(Unevictable, unevictable) __CLEARPAGEFLAG(Unevictable, unevictable) 
#ifdef CONFIG_HAVE_MLOCKED_PAGE_BIT

#define MLOCK_PAGES 1

PAGEFLAG(Mlocked, mlocked) __CLEARPAGEFLAG(Mlocked, mlocked)

	TESTSCFLAG(Mlocked, mlocked)

	TESTSCFLAG(Mlocked, mlocked) __TESTCLEARFLAG(Mlocked, mlocked)

#else

#define MLOCK_PAGES 0

PAGEFLAG_FALSE(Mlocked)

	SETPAGEFLAG_NOOP(Mlocked) TESTCLEARFLAG_FALSE(Mlocked)

PAGEFLAG_FALSE(Mlocked) SETPAGEFLAG_NOOP(Mlocked)

	TESTCLEARFLAG_FALSE(Mlocked) __TESTCLEARFLAG_FALSE(Mlocked)

#endif



#ifdef CONFIG_ARCH_USES_PG_UNCACHED

mm/page_alloc.c

+2 −2
Original line number Diff line number Diff line
@@ -557,7 +557,7 @@ static void __free_pages_ok(struct page *page, unsigned int order) 
	unsigned long flags;

	int i;

	int bad = 0;

	int wasMlocked = TestClearPageMlocked(page);

	int wasMlocked = __TestClearPageMlocked(page);



	kmemcheck_free_shadow(page, order);
@@ -1026,7 +1026,7 @@ static void free_hot_cold_page(struct page *page, int cold) 
	struct zone *zone = page_zone(page);

	struct per_cpu_pages *pcp;

	unsigned long flags;

	int wasMlocked = TestClearPageMlocked(page);

	int wasMlocked = __TestClearPageMlocked(page);



	kmemcheck_free_shadow(page, 0);