Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0b423ca2 authored by Mel Gorman's avatar Mel Gorman Committed by Linus Torvalds
Browse files

mm, page_alloc: inline pageblock lookup in page free fast paths 



The function call overhead of get_pfnblock_flags_mask() is measurable in
the page free paths.  This patch uses an inlined version that is faster.

Signed-off-by: default avatarMel Gorman <mgorman@techsingularity.net>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent e5b31ac2

include/linux/mmzone.h

+0 −7
Original line number Diff line number Diff line
@@ -85,13 +85,6 @@ extern int page_group_by_mobility_disabled; 
	get_pfnblock_flags_mask(page, page_to_pfn(page),		\

			PB_migrate_end, MIGRATETYPE_MASK)



static inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)

{

	BUILD_BUG_ON(PB_migrate_end - PB_migrate != 2);

	return get_pfnblock_flags_mask(page, pfn, PB_migrate_end,

					MIGRATETYPE_MASK);

}



struct free_area {

	struct list_head	free_list[MIGRATE_TYPES];

	unsigned long		nr_free;

mm/page_alloc.c

+100 −88
Original line number Diff line number Diff line
@@ -352,6 +352,106 @@ static inline bool update_defer_init(pg_data_t *pgdat, 
}

#endif



/* Return a pointer to the bitmap storing bits affecting a block of pages */

static inline unsigned long *get_pageblock_bitmap(struct page *page,

							unsigned long pfn)

{

#ifdef CONFIG_SPARSEMEM

	return __pfn_to_section(pfn)->pageblock_flags;

#else

	return page_zone(page)->pageblock_flags;

#endif /* CONFIG_SPARSEMEM */

}



static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)

{

#ifdef CONFIG_SPARSEMEM

	pfn &= (PAGES_PER_SECTION-1);

	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;

#else

	pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);

	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;

#endif /* CONFIG_SPARSEMEM */

}



/**

 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages

 * @page: The page within the block of interest

 * @pfn: The target page frame number

 * @end_bitidx: The last bit of interest to retrieve

 * @mask: mask of bits that the caller is interested in

 *

 * Return: pageblock_bits flags

 */

static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,

					unsigned long pfn,

					unsigned long end_bitidx,

					unsigned long mask)

{

	unsigned long *bitmap;

	unsigned long bitidx, word_bitidx;

	unsigned long word;



	bitmap = get_pageblock_bitmap(page, pfn);

	bitidx = pfn_to_bitidx(page, pfn);

	word_bitidx = bitidx / BITS_PER_LONG;

	bitidx &= (BITS_PER_LONG-1);



	word = bitmap[word_bitidx];

	bitidx += end_bitidx;

	return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;

}



unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,

					unsigned long end_bitidx,

					unsigned long mask)

{

	return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);

}



static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)

{

	return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);

}



/**

 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages

 * @page: The page within the block of interest

 * @flags: The flags to set

 * @pfn: The target page frame number

 * @end_bitidx: The last bit of interest

 * @mask: mask of bits that the caller is interested in

 */

void set_pfnblock_flags_mask(struct page *page, unsigned long flags,

					unsigned long pfn,

					unsigned long end_bitidx,

					unsigned long mask)

{

	unsigned long *bitmap;

	unsigned long bitidx, word_bitidx;

	unsigned long old_word, word;



	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);



	bitmap = get_pageblock_bitmap(page, pfn);

	bitidx = pfn_to_bitidx(page, pfn);

	word_bitidx = bitidx / BITS_PER_LONG;

	bitidx &= (BITS_PER_LONG-1);



	VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);



	bitidx += end_bitidx;

	mask <<= (BITS_PER_LONG - bitidx - 1);

	flags <<= (BITS_PER_LONG - bitidx - 1);



	word = READ_ONCE(bitmap[word_bitidx]);

	for (;;) {

		old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);

		if (word == old_word)

			break;

		word = old_word;

	}

}



void set_pageblock_migratetype(struct page *page, int migratetype)

{
@@ -6831,94 +6931,6 @@ void *__init alloc_large_system_hash(const char *tablename, 
	return table;

}



/* Return a pointer to the bitmap storing bits affecting a block of pages */

static inline unsigned long *get_pageblock_bitmap(struct page *page,

							unsigned long pfn)

{

#ifdef CONFIG_SPARSEMEM

	return __pfn_to_section(pfn)->pageblock_flags;

#else

	return page_zone(page)->pageblock_flags;

#endif /* CONFIG_SPARSEMEM */

}



static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)

{

#ifdef CONFIG_SPARSEMEM

	pfn &= (PAGES_PER_SECTION-1);

	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;

#else

	pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);

	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;

#endif /* CONFIG_SPARSEMEM */

}



/**

 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages

 * @page: The page within the block of interest

 * @pfn: The target page frame number

 * @end_bitidx: The last bit of interest to retrieve

 * @mask: mask of bits that the caller is interested in

 *

 * Return: pageblock_bits flags

 */

unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,

					unsigned long end_bitidx,

					unsigned long mask)

{

	unsigned long *bitmap;

	unsigned long bitidx, word_bitidx;

	unsigned long word;



	bitmap = get_pageblock_bitmap(page, pfn);

	bitidx = pfn_to_bitidx(page, pfn);

	word_bitidx = bitidx / BITS_PER_LONG;

	bitidx &= (BITS_PER_LONG-1);



	word = bitmap[word_bitidx];

	bitidx += end_bitidx;

	return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;

}



/**

 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages

 * @page: The page within the block of interest

 * @flags: The flags to set

 * @pfn: The target page frame number

 * @end_bitidx: The last bit of interest

 * @mask: mask of bits that the caller is interested in

 */

void set_pfnblock_flags_mask(struct page *page, unsigned long flags,

					unsigned long pfn,

					unsigned long end_bitidx,

					unsigned long mask)

{

	unsigned long *bitmap;

	unsigned long bitidx, word_bitidx;

	unsigned long old_word, word;



	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);



	bitmap = get_pageblock_bitmap(page, pfn);

	bitidx = pfn_to_bitidx(page, pfn);

	word_bitidx = bitidx / BITS_PER_LONG;

	bitidx &= (BITS_PER_LONG-1);



	VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);



	bitidx += end_bitidx;

	mask <<= (BITS_PER_LONG - bitidx - 1);

	flags <<= (BITS_PER_LONG - bitidx - 1);



	word = READ_ONCE(bitmap[word_bitidx]);

	for (;;) {

		old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);

		if (word == old_word)

			break;

		word = old_word;

	}

}



/*

 * This function checks whether pageblock includes unmovable pages or not.

 * If @count is not zero, it is okay to include less @count unmovable pages

mm/page_owner.c

+1 −1
Original line number Diff line number Diff line
@@ -143,7 +143,7 @@ print_page_owner(char __user *buf, size_t count, unsigned long pfn, 
		goto err;



	/* Print information relevant to grouping pages by mobility */

	pageblock_mt = get_pfnblock_migratetype(page, pfn);

	pageblock_mt = get_pageblock_migratetype(page);

	page_mt  = gfpflags_to_migratetype(page_ext->gfp_mask);

	ret += snprintf(kbuf + ret, count - ret,

			"PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n",

mm/vmstat.c

+1 −1
Original line number Diff line number Diff line
@@ -1041,7 +1041,7 @@ static void pagetypeinfo_showmixedcount_print(struct seq_file *m, 
		block_end_pfn = min(block_end_pfn, end_pfn);



		page = pfn_to_page(pfn);

		pageblock_mt = get_pfnblock_migratetype(page, pfn);

		pageblock_mt = get_pageblock_migratetype(page);



		for (; pfn < block_end_pfn; pfn++) {

			if (!pfn_valid_within(pfn))