Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6d777953 authored by Christoph Lameter's avatar Christoph Lameter Committed by Linus Torvalds
Browse files

mm: optimize compound_head() by avoiding a shared page flag 



The patch adds PageTail(page) and PageHead(page) to check if a page is the
head or the tail of a compound page.  This is done by masking the two bits
describing the state of a compound page and then comparing them.  So one
comparision and a branch instead of two bit checks and two branches.

Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent d85f3385

include/linux/mm.h

+2 −9
Original line number Original line Diff line number Diff line
@@ -269,14 +269,7 @@ static inline int get_page_unless_zero(struct page *page) 




static inline struct page *compound_head(struct page *page)

static inline struct page *compound_head(struct page *page)

{

{

	/*

	if (unlikely(PageTail(page)))

	 * We could avoid the PageCompound(page) check if

	 * we would not overload PageTail().

	 *

	 * This check has to be done in several performance critical

	 * paths of the slab etc. IMHO PageTail deserves its own flag.

	 */

	if (unlikely(PageCompound(page) && PageTail(page)))

		return page->first_page;

		return page->first_page;

	return page;

	return page;

}

}
@@ -327,7 +320,7 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page) 




static inline int compound_order(struct page *page)

static inline int compound_order(struct page *page)

{

{

	if (!PageCompound(page) || PageTail(page))

	if (!PageHead(page))

		return 0;

		return 0;

	return (unsigned long)page[1].lru.prev;

	return (unsigned long)page[1].lru.prev;

}

}

include/linux/page-flags.h

+26 −11
Original line number Original line Diff line number Diff line
@@ -6,6 +6,7 @@ 
#define PAGE_FLAGS_H

#define PAGE_FLAGS_H





#include <linux/types.h>

#include <linux/types.h>

#include <linux/mm_types.h>





/*

/*

 * Various page->flags bits:

 * Various page->flags bits:
@@ -94,12 +95,6 @@ 
/* PG_owner_priv_1 users should have descriptive aliases */

/* PG_owner_priv_1 users should have descriptive aliases */

#define PG_checked		PG_owner_priv_1 /* Used by some filesystems */

#define PG_checked		PG_owner_priv_1 /* Used by some filesystems */





/*

 * Marks tail portion of a compound page. We currently do not reclaim

 * compound pages so we can reuse a flag only used for reclaim here.

 */

#define PG_tail			PG_reclaim



#if (BITS_PER_LONG > 32)

#if (BITS_PER_LONG > 32)

/*

/*

 * 64-bit-only flags build down from bit 31

 * 64-bit-only flags build down from bit 31
@@ -248,12 +243,32 @@ static inline void SetPageUptodate(struct page *page) 
#define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags)

#define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags)





/*

/*

 * Note: PG_tail is an alias of another page flag. The result of PageTail()

 * PG_reclaim is used in combination with PG_compound to mark the

 * is only valid if PageCompound(page) is true.

 * head and tail of a compound page

 *

 * PG_compound & PG_reclaim	=> Tail page

 * PG_compound & ~PG_reclaim	=> Head page

 */

 */

#define PageTail(page)	test_bit(PG_tail, &(page)->flags)



#define __SetPageTail(page)	__set_bit(PG_tail, &(page)->flags)

#define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim))

#define __ClearPageTail(page)	__clear_bit(PG_tail, &(page)->flags)



#define PageTail(page)	((page->flags & PG_head_tail_mask) \

				== PG_head_tail_mask)



static inline void __SetPageTail(struct page *page)

{

	page->flags |= PG_head_tail_mask;

}



static inline void __ClearPageTail(struct page *page)

{

	page->flags &= ~PG_head_tail_mask;

}



#define PageHead(page)	((page->flags & PG_head_tail_mask) \

				== (1L << PG_compound))

#define __SetPageHead(page)	__SetPageCompound(page)

#define __ClearPageHead(page)	__ClearPageCompound(page)





#ifdef CONFIG_SWAP

#ifdef CONFIG_SWAP

#define PageSwapCache(page)	test_bit(PG_swapcache, &(page)->flags)

#define PageSwapCache(page)	test_bit(PG_swapcache, &(page)->flags)

mm/page_alloc.c

+4 −6
Original line number Original line Diff line number Diff line
@@ -235,12 +235,11 @@ static void prep_compound_page(struct page *page, unsigned long order) 




	set_compound_page_dtor(page, free_compound_page);

	set_compound_page_dtor(page, free_compound_page);

	set_compound_order(page, order);

	set_compound_order(page, order);

	__SetPageCompound(page);

	__SetPageHead(page);

	for (i = 1; i < nr_pages; i++) {

	for (i = 1; i < nr_pages; i++) {

		struct page *p = page + i;

		struct page *p = page + i;





		__SetPageTail(p);

		__SetPageTail(p);

		__SetPageCompound(p);

		p->first_page = page;

		p->first_page = page;

	}

	}

}

}
@@ -253,17 +252,16 @@ static void destroy_compound_page(struct page *page, unsigned long order) 
	if (unlikely(compound_order(page) != order))

	if (unlikely(compound_order(page) != order))

		bad_page(page);

		bad_page(page);





	if (unlikely(!PageCompound(page)))

	if (unlikely(!PageHead(page)))

			bad_page(page);

			bad_page(page);

	__ClearPageCompound(page);

	__ClearPageHead(page);

	for (i = 1; i < nr_pages; i++) {

	for (i = 1; i < nr_pages; i++) {

		struct page *p = page + i;

		struct page *p = page + i;





		if (unlikely(!PageCompound(p) | !PageTail(p) |

		if (unlikely(!PageTail(p) |

				(p->first_page != page)))

				(p->first_page != page)))

			bad_page(page);

			bad_page(page);

		__ClearPageTail(p);

		__ClearPageTail(p);

		__ClearPageCompound(p);

	}

	}

}

}